Merge changes from topic 'feat/baro_nfvi_metrics'

* changes:
  Use baro and yardstick metrics in dynamic HTML report
  benchmark.core.report: Add _combine_times
  benchmark.core.report: Add _get_baro_metrics
  Add ability to get data from different DBs in influx
  Refactor: add _format_datasets

7 files changed, 723 insertions, 71 deletions

diff --git a/api/utils/influx.py b/api/utils/influx.py
index f391ad972..8f3604745 100644
--- a/api/utils/influx.py
+++ b/api/utils/influx.py
@@ -1,5 +1,6 @@
 ##############################################################################
 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+# Copyright (c) 2019 Intel Corporation
 #
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
@@ -22,25 +23,27 @@ from yardstick import dispatcher
 
 logger = logging.getLogger(__name__)
 
-
-def get_data_db_client():
+def get_data_db_client(db=None):
     parser = ConfigParser.ConfigParser()
     try:
         parser.read(consts.CONF_FILE)
-        return _get_influxdb_client(parser)
+        return _get_influxdb_client(parser, db)
     except ConfigParser.NoOptionError:
         logger.error('Can not find the key')
         raise
 
-
-def _get_influxdb_client(parser):
+def _get_influxdb_client(parser, db=None):
     if dispatcher.INFLUXDB not in parser.get('DEFAULT', 'dispatcher'):
         raise exceptions.InfluxDBConfigurationMissing()
 
     ip = _get_ip(parser.get('dispatcher_influxdb', 'target'))
     user = parser.get('dispatcher_influxdb', 'username')
     password = parser.get('dispatcher_influxdb', 'password')
-    db_name = parser.get('dispatcher_influxdb', 'db_name')
+    if db is None:
+        db_name = parser.get('dispatcher_influxdb', 'db_name')
+    else:
+        db_name = db
+
     return influxdb_client.InfluxDBClient(ip, consts.INFLUXDB_PORT, user,
                                           password, db_name)
 
@@ -49,9 +52,9 @@ def _get_ip(url):
     return urlsplit(url).hostname
 
 
-def query(query_sql):
+def query(query_sql, db=None):
     try:
-        client = get_data_db_client()
+        client = get_data_db_client(db)
         logger.debug('Start to query: %s', query_sql)
         return list(client.query(query_sql).get_points())
     except RuntimeError:
diff --git a/yardstick/benchmark/core/report.py b/yardstick/benchmark/core/report.py
index b7d2fd02b..e5dc62050 100644
--- a/yardstick/benchmark/core/report.py
+++ b/yardstick/benchmark/core/report.py
@@ -121,6 +121,66 @@ class Report(object):
         else:
             raise KeyError("Task ID or Test case not found.")
 
+    def _get_task_start_time(self):
+        # The start time should come from the task or the metadata table.
+        # The first entry into influx for a task will be AFTER the first TC
+        # iteration
+        cmd = "select * from \"%s\" where task_id='%s' ORDER BY time ASC limit 1"
+        task_query = cmd % (self.yaml_name, self.task_id)
+
+        query_exec = influx.query(task_query)
+        start_time = query_exec[0]['time']
+        return start_time
+
+    def _get_task_end_time(self):
+        # NOTE(elfoley): when using select first() and select last() for the
+        # DB query, the timestamp returned is 0, so later queries try to
+        # return metrics from 1970
+        cmd = "select * from \"%s\" where task_id='%s' ORDER BY time DESC limit 1"
+        task_query = cmd % (self.yaml_name, self.task_id)
+        query_exec = influx.query(task_query)
+        end_time = query_exec[0]['time']
+        return end_time
+
+    def _get_baro_metrics(self):
+        start_time = self._get_task_start_time()
+        end_time = self._get_task_end_time()
+        metric_list = [
+                "cpu_value", "cpufreq_value", "intel_pmu_value",
+                 "virt_value", "memory_value"]
+        metrics = {}
+        times = []
+        query_exec = {}
+        for metric in metric_list:
+            cmd = "select * from \"%s\" where time >= '%s' and time <= '%s'"
+            query = cmd % (metric, start_time, end_time)
+            query_exec[metric] = influx.query(query, db='collectd')
+            print("query_exec: {}".format(query_exec))
+
+        for metric in query_exec:
+            print("metric in query_exec: {}".format(metric))
+            met_values = query_exec[metric]
+            print("met_values: {}".format(met_values))
+            for x in met_values:
+                x['name'] = metric
+                metric_name = str('.'.join(
+                    [x[f] for f in [
+                        'host', 'name', 'type', 'type_instance', 'instance'
+                         ] if x.get(f)]))
+
+                if not metrics.get(metric_name):
+                    metrics[metric_name] = {}
+                metric_time = self._get_trimmed_timestamp(x['time'])
+                times.append(metric_time)
+                time = metric_time
+                metrics[metric_name][time] = x['value']
+
+        times = sorted(list(set(times)))
+
+        metrics['Timestamp'] = times
+        print("metrics: {}".format(metrics))
+        return metrics
+
     def _get_trimmed_timestamp(self, metric_time, resolution=4):
         if not isinstance(metric_time, str):
             metric_time = metric_time.encode('utf8') # PY2: unicode to str
@@ -138,6 +198,46 @@ class Report(object):
             timestamps.append(metric_time)               # HH:MM:SS.micros
         return timestamps
 
+    def _format_datasets(self, metric_name, metrics):
+        values = []
+        for metric in metrics:
+            val = metric.get(metric_name, None)
+            if val is None:
+                # keep explicit None or missing entry as is
+                pass
+            elif isinstance(val, (int, float)):
+                # keep plain int or float as is
+                pass
+            elif six.PY2 and isinstance(val,
+                        long):  # pylint: disable=undefined-variable
+                # PY2: long value would be rendered with trailing L,
+                # which JS does not support, so convert it to float
+                val = float(val)
+            elif isinstance(val, six.string_types):
+                s = val
+                if not isinstance(s, str):
+                    s = s.encode('utf8')            # PY2: unicode to str
+                try:
+                    # convert until failure
+                    val = s
+                    val = float(s)
+                    val = int(s)
+                    if six.PY2 and isinstance(val,
+                                long):  # pylint: disable=undefined-variable
+                        val = float(val)            # PY2: long to float
+                except ValueError:
+                    # value may have been converted to a number
+                    pass
+                finally:
+                    # if val was not converted into a num, then it must be
+                    # text, which shouldn't end up in the report
+                    if isinstance(val, six.string_types):
+                        val = None
+            else:
+                raise ValueError("Cannot convert %r" % val)
+            values.append(val)
+        return values
+
     @cliargs("task_id", type=str, help=" task id", nargs=1)
     @cliargs("yaml_name", type=str, help=" Yaml file Name", nargs=1)
     def _generate_common(self, args):
@@ -174,37 +274,7 @@ class Report(object):
 
         # extract and convert field values
         for key in field_keys:
-            values = []
-            for metric in db_metrics:
-                val = metric.get(key, None)
-                if val is None:
-                    # keep explicit None or missing entry as is
-                    pass
-                elif isinstance(val, (int, float)):
-                    # keep plain int or float as is
-                    pass
-                elif six.PY2 and isinstance(val,
-                            long):  # pylint: disable=undefined-variable
-                    # PY2: long value would be rendered with trailing L,
-                    # which JS does not support, so convert it to float
-                    val = float(val)
-                elif isinstance(val, six.string_types):
-                    s = val
-                    if not isinstance(s, str):
-                        s = s.encode('utf8')            # PY2: unicode to str
-                    try:
-                        # convert until failure
-                        val = s
-                        val = float(s)
-                        val = int(s)
-                        if six.PY2 and isinstance(val,
-                                    long):  # pylint: disable=undefined-variable
-                            val = float(val)            # PY2: long to float
-                    except ValueError:
-                        pass
-                else:
-                    raise ValueError("Cannot convert %r" % val)
-                values.append(val)
+            values = self._format_datasets(key, db_metrics)
             datasets.append({'label': key, 'data': values})
             table_vals[key] = values
 
@@ -235,31 +305,80 @@ class Report(object):
 
         print("Report generated. View %s" % consts.DEFAULT_HTML_FILE)
 
+    def _combine_times(self, *args):
+        times = []
+        # Combines an arbitrary number of lists
+        [times.extend(x) for x in args]
+        times = list(set(times))
+        times.sort()
+        return times
+
+    def _combine_metrics(self, *args):
+        baro_data, baro_time, yard_data, yard_time = args
+        combo_time = self._combine_times(baro_time, yard_time)
+
+        data = {}
+        [data.update(x) for x in (baro_data, yard_data)]
+
+        table_data = {}
+        table_data['Timestamp'] = combo_time
+        combo = {}
+        keys = sorted(data.keys())
+        for met_name in data:
+            dataset = []
+            for point in data[met_name]:
+                 dataset.append({'x': point, 'y': data[met_name][point]})
+            # the metrics need to be ordered by time
+            combo[met_name] = sorted(dataset, key=lambda i: i['x'])
+        for met_name in data:
+            table_data[met_name] = []
+            for t in combo_time:
+                table_data[met_name].append(data[met_name].get(t, ''))
+        return combo, keys, table_data
+
     @cliargs("task_id", type=str, help=" task id", nargs=1)
     @cliargs("yaml_name", type=str, help=" Yaml file Name", nargs=1)
     def generate_nsb(self, args):
         """Start NSB report generation."""
         _, report_data = self._generate_common(args)
         report_time = report_data.pop('Timestamp')
-        report_keys = sorted(report_data, key=str.lower)
-        report_tree = JSTree().format_for_jstree(report_keys)
         report_meta = {
             "testcase": self.yaml_name,
             "task_id": self.task_id,
         }
 
+        yardstick_data = {}
+        for i, t in enumerate(report_time):
+            for m in report_data:
+                if not yardstick_data.get(m):
+                   yardstick_data[m] = {}
+                yardstick_data[m][t] = report_data[m][i]
+
+        baro_data = self._get_baro_metrics()
+        baro_timestamps = baro_data.pop('Timestamp')
+
+        yard_timestamps = report_time
+        report_time = self._combine_times(yard_timestamps, baro_timestamps)
+
+        combo_metrics, combo_keys, combo_table = self._combine_metrics(
+            baro_data, baro_timestamps, yardstick_data, yard_timestamps)
+        combo_time = self._combine_times(baro_timestamps, yard_timestamps)
+        combo_tree = JSTree().format_for_jstree(combo_keys)
+
         template_dir = consts.YARDSTICK_ROOT_PATH + "yardstick/common"
         template_environment = jinja2.Environment(
             autoescape=False,
             loader=jinja2.FileSystemLoader(template_dir),
             lstrip_blocks=True)
 
+        combo_data = combo_metrics
         context = {
             "report_meta": report_meta,
-            "report_data": report_data,
-            "report_time": report_time,
-            "report_keys": report_keys,
-            "report_tree": report_tree,
+            "report_data": combo_data,
+            "report_time": combo_time,
+            "report_keys": combo_keys,
+            "report_tree": combo_tree,
+            "table_data": combo_table,
         }
 
         template_html = template_environment.get_template("nsb_report.html.j2")
diff --git a/yardstick/common/nsb_report.html.j2 b/yardstick/common/nsb_report.html.j2
index aa90253f8..a6713eb16 100644
--- a/yardstick/common/nsb_report.html.j2
+++ b/yardstick/common/nsb_report.html.j2
@@ -3,7 +3,7 @@
 
 <!--
  Copyright (c) 2017 Rajesh Kudaka <4k.rajesh@gmail.com>
- Copyright (c) 2018 Intel Corporation.
+ Copyright (c) 2018-2019 Intel Corporation.
 
  All rights reserved. This program and the accompanying materials
  are made available under the terms of the Apache License, Version 2.0
@@ -57,6 +57,7 @@
             var report_time = {{report_time|safe}};
             var report_keys = {{report_keys|safe}};
             var report_tree = {{report_tree|safe}};
+            var table_data = {{table_data|safe}};
 
             // Wait for DOM to be loaded
             $(function() {
@@ -64,10 +65,10 @@
                 var cnvGraph = $('#cnvGraph');
                 var divTree = $('#divTree');
 
-                create_table(tblMetrics, report_data, report_time, report_keys);
+                create_table(tblMetrics, table_data, report_time, report_keys);
                 var objGraph = create_graph(cnvGraph, report_time);
                 create_tree(divTree, report_tree);
-                handle_tree(divTree, tblMetrics, objGraph, report_data, report_time);
+                handle_tree(divTree, tblMetrics, objGraph, report_data, table_data, report_time);
             });
         </script>
     </body>
diff --git a/yardstick/common/nsb_report.js b/yardstick/common/nsb_report.js
index 4de1c8e78..18141900b 100644
--- a/yardstick/common/nsb_report.js
+++ b/yardstick/common/nsb_report.js
@@ -1,6 +1,6 @@
 /*******************************************************************************
  * Copyright (c) 2017 Rajesh Kudaka <4k.rajesh@gmail.com>
- * Copyright (c) 2018 Intel Corporation.
+ * Copyright (c) 2018-2019 Intel Corporation.
  *
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Apache License, Version 2.0
@@ -72,11 +72,16 @@ function create_graph(cnvGraph, timestamps)
                     borderWidth: 2,
                     fill: false,
                     tension: 0,
+                    showline: true,
+                    spanGaps: true,
                 },
             },
             scales: {
                 xAxes: [{
                     type: 'category',
+                    display: true,
+                    labels: timestamps,
+                    autoSkip: true,
                 }],
                 yAxes: [{
                     type: 'linear',
@@ -144,7 +149,7 @@ function update_graph(objGraph, datasets)
     objGraph.update();
 }
 
-function handle_tree(divTree, tblMetrics, objGraph, table_data, timestamps)
+function handle_tree(divTree, tblMetrics, objGraph, graph_data, table_data, timestamps)
 {
     divTree.on('check_node.jstree uncheck_node.jstree', function(e, data) {
         var selected_keys = [];
@@ -155,7 +160,7 @@ function handle_tree(divTree, tblMetrics, objGraph, table_data, timestamps)
                 selected_keys.push(node.id);
                 selected_datasets.push({
                     label: node.id,
-                    data: table_data[node.id],
+                    data: graph_data[node.id],
                 });
             }
         });
diff --git a/yardstick/tests/functional/benchmark/core/test_report.py b/yardstick/tests/functional/benchmark/core/test_report.py
index 5f060dd1e..832d3b3e1 100644
--- a/yardstick/tests/functional/benchmark/core/test_report.py
+++ b/yardstick/tests/functional/benchmark/core/test_report.py
@@ -1,5 +1,5 @@
 ##############################################################################
-# Copyright (c) 2018 Intel Corporation.
+# Copyright (c) 2018-2019 Intel Corporation.
 #
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
@@ -41,6 +41,33 @@ GOOD_DB_METRICS = [
     {u'time': u'2018-08-20T16:49:30.379359421Z',
      u'metric1': 8, u'metric2': 5, u'metric3': 1, u'metric4': 0},
 ]
+GOOD_DB_BARO_METRICS = [
+     {u'value': 324050, u'instance': u'0', u'host': u'myhostname',
+      u'time': u'2018-08-20T16:49:27.383698038Z',
+      u'type_instance': u'user', u'type': u'cpu'},
+     {
+      u'value': 193798, u'instance': u'0', u'host': u'myhostname',
+      u'time': u'2018-12-19T16:49:27.383712594Z',
+      u'type_instance': u'system', u'type': u'cpu'},
+     {
+      u'value': 324051, u'instance': u'0', u'host': u'myhostname',
+      u'time': u'2018-08-20T16:49:28.383696624Z',
+      u'type_instance': u'user', u'type': u'cpu'},
+     {
+      u'value': 193800, u'instance': u'0', u'host': u'myhostname',
+      u'time': u'2018-08-20T16:49:28.383713481Z',
+      u'type_instance': u'system', u'type': u'cpu'},
+     {
+      u'value': 324054, u'instance': u'0', u'host': u'myhostname',
+      u'time': u'2018-08-20T16:49:29.3836966789Z',
+      u'type_instance': u'user', u'type': u'cpu'},
+     {
+      u'value': 193801, u'instance': u'0', u'host': u'myhostname',
+      u'time': u'2018-08-20T16:49:29.383716296Z',
+      u'type_instance': u'system', u'type': u'cpu'}
+]
+TIMESTAMP_START = '2018-08-20T16:49:26.372662016Z'
+TIMESTAMP_END = '2018-08-20T16:49:30.379359421Z'
 
 yardstick_config = """
 [DEFAULT]
@@ -48,11 +75,19 @@ dispatcher = influxdb
 """
 
 
-def my_query(query_sql):
+def my_query(query_sql, db=None):
     get_fieldkeys_cmd = 'show field keys'
     get_metrics_cmd = 'select * from'
-
-    if get_fieldkeys_cmd in query_sql:
+    get_start_time_cmd = 'ORDER ASC limit 1'
+    get_end_time_cmd = 'ORDER DESC limit 1'
+    if db:
+        if get_start_time_cmd in query_sql:
+            return TIMESTAMP_START
+        elif get_end_time_cmd in query_sql:
+            return TIMESTAMP_END
+        else:
+            return GOOD_DB_BARO_METRICS
+    elif get_fieldkeys_cmd in query_sql:
         return GOOD_DB_FIELDKEYS
     elif get_metrics_cmd in query_sql:
         return GOOD_DB_METRICS
@@ -87,25 +122,190 @@ class ReportTestCase(unittest.TestCase):
                      keys_act = ast.literal_eval(l.strip()[18:-1])
                  elif "var report_tree = [" in l:
                      tree_act = ast.literal_eval(l.strip()[18:-1])
-
         data_exp = {
-            'metric1': [1, 1, 2, 3, 5, 8],
-            'metric2': [0, 1, 2, 3, 4, 5],
-            'metric3': [8, 5, 3, 2, 1, 1],
-            'metric4': [5, 4, 3, 2, 1, 0],
+            'metric1': [
+                {'x': '16:49:26.372662', 'y': 1},
+                {'x': '16:49:27.374208', 'y': 1},
+                {'x': '16:49:28.375742', 'y': 2},
+                {'x': '16:49:29.377299', 'y': 3},
+                {'x': '16:49:30.378252', 'y': 5},
+                {'x': '16:49:30.379359', 'y': 8}],
+            'metric2': [
+                {'x': '16:49:26.372662', 'y': 0},
+                {'x': '16:49:27.374208', 'y': 1},
+                {'x': '16:49:28.375742', 'y': 2},
+                {'x': '16:49:29.377299', 'y': 3},
+                {'x': '16:49:30.378252', 'y': 4},
+                {'x': '16:49:30.379359', 'y': 5}],
+            'metric3': [
+                {'x': '16:49:26.372662', 'y': 8},
+                {'x': '16:49:27.374208', 'y': 5},
+                {'x': '16:49:28.375742', 'y': 3},
+                {'x': '16:49:29.377299', 'y': 2},
+                {'x': '16:49:30.378252', 'y': 1},
+                {'x': '16:49:30.379359', 'y': 1}],
+            'metric4': [
+                {'x': '16:49:26.372662', 'y': 5},
+                {'x': '16:49:27.374208', 'y': 4},
+                {'x': '16:49:28.375742', 'y': 3},
+                {'x': '16:49:29.377299', 'y': 2},
+                {'x': '16:49:30.378252', 'y': 1},
+                {'x': '16:49:30.379359', 'y': 0}],
+            'myhostname.cpu_value.cpu.system.0': [
+                {'x': '16:49:27.3837', 'y': 193798},
+                {'x': '16:49:28.3837', 'y': 193800},
+                {'x': '16:49:29.3837', 'y': 193801}],
+            'myhostname.cpu_value.cpu.user.0': [
+                {'x': '16:49:27.3836', 'y': 324050},
+                {'x': '16:49:28.3836', 'y': 324051},
+                {'x': '16:49:29.3836', 'y': 324054}],
+            'myhostname.cpufreq_value.cpu.system.0': [
+                {'x': '16:49:27.3837', 'y': 193798},
+                {'x': '16:49:28.3837', 'y': 193800},
+                {'x': '16:49:29.3837', 'y': 193801}],
+            'myhostname.cpufreq_value.cpu.user.0': [
+                {'x': '16:49:27.3836', 'y': 324050},
+                {'x': '16:49:28.3836', 'y': 324051},
+                {'x': '16:49:29.3836', 'y': 324054}],
+            'myhostname.intel_pmu_value.cpu.system.0': [
+                {'x': '16:49:27.3837', 'y': 193798},
+                {'x': '16:49:28.3837', 'y': 193800},
+                {'x': '16:49:29.3837', 'y': 193801}],
+            'myhostname.intel_pmu_value.cpu.user.0': [
+                {'x': '16:49:27.3836', 'y': 324050},
+                {'x': '16:49:28.3836', 'y': 324051},
+                {'x': '16:49:29.3836', 'y': 324054}],
+            'myhostname.virt_value.cpu.system.0': [
+                {'x': '16:49:27.3837', 'y': 193798},
+                {'x': '16:49:28.3837', 'y': 193800},
+                {'x': '16:49:29.3837', 'y': 193801}],
+            'myhostname.virt_value.cpu.user.0': [
+                {'x': '16:49:27.3836', 'y': 324050},
+                {'x': '16:49:28.3836', 'y': 324051},
+                {'x': '16:49:29.3836', 'y': 324054}],
+            'myhostname.memory_value.cpu.system.0': [
+                {'x': '16:49:27.3837', 'y': 193798},
+                {'x': '16:49:28.3837', 'y': 193800},
+                {'x': '16:49:29.3837', 'y': 193801}],
+            'myhostname.memory_value.cpu.user.0': [
+                {'x': '16:49:27.3836', 'y': 324050},
+                {'x': '16:49:28.3836', 'y': 324051},
+                {'x': '16:49:29.3836', 'y': 324054}]
         }
         time_exp = [
-            '16:49:26.372662', '16:49:27.374208', '16:49:28.375742',
-            '16:49:29.377299', '16:49:30.378252', '16:49:30.379359',
+            '16:49:26.372662', '16:49:27.374208', '16:49:27.3836',
+            '16:49:27.3837', '16:49:28.375742', '16:49:28.3836',
+            '16:49:28.3837', '16:49:29.377299', '16:49:29.3836',
+            '16:49:29.3837', '16:49:30.378252', '16:49:30.379359',
         ]
-        keys_exp = [
+        keys_exp = sorted([
             'metric1', 'metric2', 'metric3', 'metric4',
-        ]
+            'myhostname.cpu_value.cpu.system.0',
+            'myhostname.cpu_value.cpu.user.0',
+            'myhostname.cpufreq_value.cpu.system.0',
+            'myhostname.cpufreq_value.cpu.user.0',
+            'myhostname.intel_pmu_value.cpu.system.0',
+            'myhostname.intel_pmu_value.cpu.user.0',
+            'myhostname.virt_value.cpu.system.0',
+            'myhostname.virt_value.cpu.user.0',
+            'myhostname.memory_value.cpu.system.0',
+            'myhostname.memory_value.cpu.user.0',
+        ])
         tree_exp = [
             {'parent': '#', 'text': 'metric1', 'id': 'metric1'},
             {'parent': '#', 'text': 'metric2', 'id': 'metric2'},
             {'parent': '#', 'text': 'metric3', 'id': 'metric3'},
             {'parent': '#', 'text': 'metric4', 'id': 'metric4'},
+            {'id': 'myhostname', 'parent': '#', 'text': 'myhostname'},
+            {'id': 'myhostname.cpu_value',
+             'parent': 'myhostname',
+             'text': 'cpu_value'},
+            {'id': 'myhostname.cpu_value.cpu',
+             'parent': 'myhostname.cpu_value',
+             'text': 'cpu'},
+            {'id': 'myhostname.cpu_value.cpu.system',
+             'parent': 'myhostname.cpu_value.cpu',
+             'text': 'system'},
+            {'id': 'myhostname.cpu_value.cpu.system.0',
+             'parent': 'myhostname.cpu_value.cpu.system',
+             'text': '0'},
+            {'id': 'myhostname.cpu_value.cpu.user',
+             'parent': 'myhostname.cpu_value.cpu',
+             'text': 'user'},
+            {'id': 'myhostname.cpu_value.cpu.user.0',
+             'parent': 'myhostname.cpu_value.cpu.user',
+             'text': '0'},
+            {'id': 'myhostname.cpufreq_value',
+             'parent': 'myhostname',
+             'text': 'cpufreq_value'},
+            {'id': 'myhostname.cpufreq_value.cpu',
+             'parent': 'myhostname.cpufreq_value',
+             'text': 'cpu'},
+            {'id': 'myhostname.cpufreq_value.cpu.system',
+             'parent': 'myhostname.cpufreq_value.cpu',
+             'text': 'system'},
+            {'id': 'myhostname.cpufreq_value.cpu.system.0',
+             'parent': 'myhostname.cpufreq_value.cpu.system',
+             'text': '0'},
+            {'id': 'myhostname.cpufreq_value.cpu.user',
+             'parent': 'myhostname.cpufreq_value.cpu',
+             'text': 'user'},
+            {'id': 'myhostname.cpufreq_value.cpu.user.0',
+             'parent': 'myhostname.cpufreq_value.cpu.user',
+             'text': '0'},
+            {'id': 'myhostname.intel_pmu_value',
+             'parent': 'myhostname',
+             'text': 'intel_pmu_value'},
+            {'id': 'myhostname.intel_pmu_value.cpu',
+             'parent': 'myhostname.intel_pmu_value',
+             'text': 'cpu'},
+            {'id': 'myhostname.intel_pmu_value.cpu.system',
+             'parent': 'myhostname.intel_pmu_value.cpu',
+             'text': 'system'},
+            {'id': 'myhostname.intel_pmu_value.cpu.system.0',
+             'parent': 'myhostname.intel_pmu_value.cpu.system',
+             'text': '0'},
+            {'id': 'myhostname.intel_pmu_value.cpu.user',
+             'parent': 'myhostname.intel_pmu_value.cpu',
+             'text': 'user'},
+            {'id': 'myhostname.intel_pmu_value.cpu.user.0',
+             'parent': 'myhostname.intel_pmu_value.cpu.user',
+             'text': '0'},
+            {'id': 'myhostname.memory_value',
+             'parent': 'myhostname',
+             'text': 'memory_value'},
+            {'id': 'myhostname.memory_value.cpu',
+             'parent': 'myhostname.memory_value',
+             'text': 'cpu'},
+            {'id': 'myhostname.memory_value.cpu.system',
+             'parent': 'myhostname.memory_value.cpu',
+             'text': 'system'},
+            {'id': 'myhostname.memory_value.cpu.system.0',
+             'parent': 'myhostname.memory_value.cpu.system',
+             'text': '0'},
+            {'id': 'myhostname.memory_value.cpu.user',
+             'parent': 'myhostname.memory_value.cpu',
+             'text': 'user'},
+            {'id': 'myhostname.memory_value.cpu.user.0',
+             'parent': 'myhostname.memory_value.cpu.user',
+             'text': '0'},
+            {'id': 'myhostname.virt_value', 'parent': 'myhostname',
+             'text': 'virt_value'},
+            {'id': 'myhostname.virt_value.cpu',
+             'parent': 'myhostname.virt_value',
+             'text': 'cpu'},
+            {'id': 'myhostname.virt_value.cpu.system',
+             'parent': 'myhostname.virt_value.cpu',
+             'text': 'system'},
+            {'id': 'myhostname.virt_value.cpu.system.0',
+             'parent': 'myhostname.virt_value.cpu.system',
+             'text': '0'},
+            {'id': 'myhostname.virt_value.cpu.user',
+             'parent': 'myhostname.virt_value.cpu',
+             'text': 'user'},
+            {'id': 'myhostname.virt_value.cpu.user.0',
+             'parent': 'myhostname.virt_value.cpu.user',
+             'text': '0'}
         ]
 
         self.assertEqual(data_exp, data_act)
diff --git a/yardstick/tests/unit/apiserver/utils/test_influx.py b/yardstick/tests/unit/apiserver/utils/test_influx.py
index 6021d35df..3a97ff292 100644
--- a/yardstick/tests/unit/apiserver/utils/test_influx.py
+++ b/yardstick/tests/unit/apiserver/utils/test_influx.py
@@ -1,5 +1,6 @@
 ##############################################################################
 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+# Copyright (c) 2019 Intel Corporation.
 #
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
@@ -29,7 +30,7 @@ class GetDataDbClientTestCase(base.BaseUnitTestCase):
 
         self.assertEqual('fake_client', influx.get_data_db_client())
         _mock_parser.read.assert_called_once_with(constants.CONF_FILE)
-        mock_get_client.assert_called_once_with(_mock_parser)
+        mock_get_client.assert_called_once_with(_mock_parser, None)
 
     @mock.patch.object(influx.logger, 'error')
     @mock.patch.object(influx, '_get_influxdb_client',
@@ -46,7 +47,7 @@ class GetDataDbClientTestCase(base.BaseUnitTestCase):
             influx.get_data_db_client()
 
         _mock_parser.read.assert_called_once_with(constants.CONF_FILE)
-        mock_get_client.assert_called_once_with(_mock_parser)
+        mock_get_client.assert_called_once_with(_mock_parser, None)
 
 
 class GetIpTestCase(base.BaseUnitTestCase):
diff --git a/yardstick/tests/unit/benchmark/core/test_report.py b/yardstick/tests/unit/benchmark/core/test_report.py
index b498299a9..89fb1e90a 100644
--- a/yardstick/tests/unit/benchmark/core/test_report.py
+++ b/yardstick/tests/unit/benchmark/core/test_report.py
@@ -77,10 +77,10 @@ MORE_EXPECTED_TABLE_VALS = {
             123,
             4.56,
             9876543210987654321 if six.PY3 else 9.876543210987655e+18,
-            'str_str value',
-            'str_unicode value',
-            'unicode_str value',
-            'unicode_unicode value',
+            None,
+            None,
+            None,
+            None,
             7.89,
             1011,
             9876543210123456789 if six.PY3 else 9.876543210123457e+18,
@@ -219,6 +219,114 @@ class ReportTestCase(unittest.TestCase):
         self.rep.task_id = GOOD_TASK_ID
         six.assertRaisesRegex(self, KeyError, "Task ID", self.rep._get_metrics)
 
+    @mock.patch.object(influx, 'query')
+    def test__get_task_start_time(self, mock_query):
+        self.rep.yaml_name = GOOD_YAML_NAME
+        self.rep.task_id = GOOD_TASK_ID
+        mock_query.return_value = [{
+            u'free.memory0.used': u'9789088',
+            u'free.memory0.available': u'22192984',
+            u'free.memory0.shared': u'219152',
+            u'time': u'2019-01-22T16:20:14.568075776Z',
+        }]
+        expected = "2019-01-22T16:20:14.568075776Z"
+
+        self.assertEqual(
+            expected,
+            self.rep._get_task_start_time()
+        )
+
+    def test__get_task_start_time_task_not_found(self):
+        pass
+
+    @mock.patch.object(influx, 'query')
+    def test__get_task_end_time(self, mock_query):
+        self.rep.yaml_name = GOOD_YAML_NAME
+        self.rep.task_id = GOOD_TASK_ID
+        # TODO(elfoley): write this test!
+        mock_query.return_value = [{
+
+        }]
+
+    @mock.patch.object(influx, 'query')
+    def test__get_baro_metrics(self, mock_query):
+        self.rep.yaml_name = GOOD_YAML_NAME
+        self.rep.task_id = GOOD_TASK_ID
+        self.rep._get_task_start_time = mock.Mock(return_value=0)
+        self.rep._get_task_end_time = mock.Mock(return_value=0)
+
+        influx_return_values = ([{
+             u'value': 324050, u'instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:25.383698038Z',
+             u'type_instance': u'user', u'type': u'cpu',
+             }, {
+             u'value': 193798, u'instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:25.383712594Z',
+             u'type_instance': u'system', u'type': u'cpu',
+             }, {
+             u'value': 324051, u'instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:35.383696624Z',
+             u'type_instance': u'user', u'type': u'cpu',
+             }, {
+             u'value': 193800, u'instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:35.383713481Z',
+             u'type_instance': u'system', u'type': u'cpu',
+             }, {
+             u'value': 324054, u'instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:45.3836966789Z',
+             u'type_instance': u'user', u'type': u'cpu',
+             }, {
+             u'value': 193801, u'instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:45.383716296Z',
+             u'type_instance': u'system', u'type': u'cpu',
+             }],
+             [{
+             u'value': 3598453000, u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:25.383698038Z',
+             u'type_instance': u'0', u'type': u'cpufreq',
+             }, {
+             u'value': 3530250000, u'type_instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:35.383712594Z', u'type': u'cpufreq',
+             }, {
+             u'value': 3600281000, u'type_instance': u'0', u'host': u'myhostname',
+             u'time': u'2018-12-19T14:11:45.383696624Z', u'type': u'cpufreq',
+            }],
+        )
+
+        def ret_vals(vals):
+            for x in vals:
+                yield x
+            while True:
+                yield []
+
+        mock_query.side_effect = ret_vals(influx_return_values)
+
+        BARO_EXPECTED_METRICS = {
+            'Timestamp': [
+                '14:11:25.3836', '14:11:25.3837',
+                '14:11:35.3836', '14:11:35.3837',
+                '14:11:45.3836', '14:11:45.3837'],
+            'myhostname.cpu_value.cpu.user.0': {
+                '14:11:25.3836': 324050,
+                '14:11:35.3836': 324051,
+                '14:11:45.3836': 324054,
+            },
+            'myhostname.cpu_value.cpu.system.0': {
+                '14:11:25.3837': 193798,
+                '14:11:35.3837': 193800,
+                '14:11:45.3837': 193801,
+            },
+            'myhostname.cpufreq_value.cpufreq.0': {
+                '14:11:25.3836': 3598453000,
+                '14:11:35.3837': 3530250000,
+                '14:11:45.3836': 3600281000,
+            }
+        }
+        self.assertEqual(
+            BARO_EXPECTED_METRICS,
+            self.rep._get_baro_metrics()
+        )
+
     def test__get_timestamps(self):
 
         metrics = MORE_DB_METRICS
@@ -227,6 +335,200 @@ class ReportTestCase(unittest.TestCase):
             self.rep._get_timestamps(metrics)
         )
 
+    def test__format_datasets(self):
+        metric_name = "free.memory0.used"
+        metrics = [{
+            u'free.memory1.free': u'1958664',
+            u'free.memory0.used': u'9789560',
+            }, {
+            u'free.memory1.free': u'1958228',
+            u'free.memory0.used': u'9789790',
+            }, {
+            u'free.memory1.free': u'1956156',
+            u'free.memory0.used': u'9791092',
+            }, {
+            u'free.memory1.free': u'1956280',
+            u'free.memory0.used': u'9790796',
+        }]
+        self.assertEqual(
+            [9789560, 9789790, 9791092, 9790796,],
+            self.rep._format_datasets(metric_name, metrics)
+        )
+
+    def test__format_datasets_val_none(self):
+         metric_name = "free.memory0.used"
+         metrics = [{
+            u'free.memory1.free': u'1958664',
+            u'free.memory0.used': 9876543109876543210,
+            }, {
+            u'free.memory1.free': u'1958228',
+            }, {
+            u'free.memory1.free': u'1956156',
+            u'free.memory0.used': u'9791092',
+            }, {
+            u'free.memory1.free': u'1956280',
+            u'free.memory0.used': u'9790796',
+         }]
+
+         exp0 = 9876543109876543210 if six.PY3 else 9.876543109876543e+18
+         self.assertEqual(
+            [exp0, None, 9791092, 9790796],
+            self.rep._format_datasets(metric_name, metrics)
+         )
+
+    def test__format_datasets_val_incompatible(self):
+        metric_name = "free.memory0.used"
+        metrics = [{
+            u'free.memory0.used': "some incompatible value",
+            }, {
+        }]
+        self.assertEqual(
+            [None, None],
+            self.rep._format_datasets(metric_name, metrics)
+        )
+
+    def test__combine_times(self):
+        yard_times = [
+            '00:00:00.000000',
+            '00:00:01.000000',
+            '00:00:02.000000',
+            '00:00:06.000000',
+            '00:00:08.000000',
+            '00:00:09.000000',
+        ]
+        baro_times = [
+            '00:00:01.000000',
+            '00:00:03.000000',
+            '00:00:04.000000',
+            '00:00:05.000000',
+            '00:00:07.000000',
+            '00:00:10.000000',
+        ]
+        expected_combo = [
+            '00:00:00.000000',
+            '00:00:01.000000',
+            '00:00:02.000000',
+            '00:00:03.000000',
+            '00:00:04.000000',
+            '00:00:05.000000',
+            '00:00:06.000000',
+            '00:00:07.000000',
+            '00:00:08.000000',
+            '00:00:09.000000',
+            '00:00:10.000000',
+        ]
+
+        actual_combo = self.rep._combine_times(yard_times, baro_times)
+        self.assertEqual(len(expected_combo), len(actual_combo))
+
+        self.assertEqual(
+            expected_combo,
+            actual_combo,
+        )
+
+    def test__combine_times_2(self):
+        time1 = ['14:11:25.383698', '14:11:25.383712', '14:11:35.383696',]
+        time2 = [
+            '16:20:14.568075', '16:20:24.575083',
+            '16:20:34.580989', '16:20:44.586801', ]
+        time_exp = [
+            '14:11:25.383698', '14:11:25.383712', '14:11:35.383696',
+            '16:20:14.568075', '16:20:24.575083', '16:20:34.580989',
+            '16:20:44.586801',
+        ]
+        self.assertEqual(time_exp, self.rep._combine_times(time1, time2))
+
+    def test__combine_metrics(self):
+        BARO_METRICS = {
+            'myhostname.cpu_value.cpu.user.0': {
+                '14:11:25.3836': 324050, '14:11:35.3836': 324051,
+                '14:11:45.3836': 324054,
+            },
+            'myhostname.cpu_value.cpu.system.0': {
+                '14:11:25.3837': 193798, '14:11:35.3837': 193800,
+                '14:11:45.3837': 193801,
+            }
+        }
+        BARO_TIMES = [
+            '14:11:25.3836', '14:11:25.3837', '14:11:35.3836',
+            '14:11:35.3837', '14:11:45.3836', '14:11:45.3837',
+        ]
+        YARD_METRICS = {
+            'free.memory9.free': {
+                '16:20:14.5680': 1958244, '16:20:24.5750': 1955964,
+                '16:20:34.5809': 1956040, '16:20:44.5868': 1956428,
+            },
+            'free.memory7.used': {
+                '16:20:14.5680': 9789068, '16:20:24.5750': 9791284,
+                '16:20:34.5809': 9791228, '16:20:44.5868': 9790692,
+            },
+            'free.memory2.total':{
+                '16:20:14.5680': 32671288, '16:20:24.5750': 32671288,
+                '16:20:34.5809': 32671288, '16:20:44.5868': 32671288,
+            },
+            'free.memory7.free': {
+                '16:20:14.5680': 1958368, '16:20:24.5750': 1956104,
+                '16:20:34.5809': 1956040, '16:20:44.5868': 1956552,
+            },
+            'free.memory1.used': {
+                '16:20:14.5680': 9788872, '16:20:24.5750': 9789212,
+                '16:20:34.5809': 9791168, '16:20:44.5868': 9790996,
+            },
+        }
+        YARD_TIMES = [
+             '16:20:14.5680', '16:20:24.5750',
+             '16:20:34.5809', '16:20:44.5868',
+        ]
+
+        expected_output = {
+            'myhostname.cpu_value.cpu.user.0': [{
+                'x': '14:11:25.3836', 'y': 324050, }, {
+                'x': '14:11:35.3836', 'y': 324051, }, {
+                'x': '14:11:45.3836', 'y': 324054, }],
+            'myhostname.cpu_value.cpu.system.0' : [{
+                'x': '14:11:25.3837', 'y': 193798, }, {
+                'x': '14:11:35.3837', 'y': 193800, }, {
+                'x': '14:11:45.3837', 'y': 193801, }],
+            'free.memory9.free': [{
+                'x': '16:20:14.5680', 'y': 1958244, }, {
+                'x': '16:20:24.5750', 'y': 1955964, }, {
+                'x': '16:20:34.5809', 'y': 1956040, }, {
+                'x': '16:20:44.5868', 'y': 1956428, }],
+            'free.memory7.used': [{
+                'x': '16:20:14.5680', 'y': 9789068, }, {
+                'x': '16:20:24.5750', 'y': 9791284, }, {
+                'x': '16:20:34.5809', 'y': 9791228, }, {
+                'x': '16:20:44.5868', 'y': 9790692, }],
+            'free.memory2.total': [{
+                'x': '16:20:14.5680', 'y': 32671288, }, {
+                'x': '16:20:24.5750', 'y': 32671288, }, {
+                'x': '16:20:34.5809', 'y': 32671288, }, {
+                'x': '16:20:44.5868', 'y': 32671288, }],
+            'free.memory7.free': [{
+                'x': '16:20:14.5680', 'y': 1958368, }, {
+                'x': '16:20:24.5750', 'y': 1956104, }, {
+                'x': '16:20:34.5809', 'y': 1956040, }, {
+                'x': '16:20:44.5868', 'y': 1956552, }],
+           'free.memory1.used': [{
+                'x': '16:20:14.5680', 'y': 9788872, }, {
+                'x': '16:20:24.5750', 'y': 9789212, }, {
+                'x': '16:20:34.5809', 'y': 9791168, }, {
+                'x': '16:20:44.5868', 'y': 9790996, }],
+           }
+
+        actual_output, _, _ = self.rep._combine_metrics(
+            BARO_METRICS, BARO_TIMES, YARD_METRICS, YARD_TIMES
+        )
+        self.assertEquals(
+            sorted(expected_output.keys()),
+            sorted(actual_output.keys())
+        )
+
+        self.assertEquals(
+            expected_output,
+            actual_output,
+        )
+
     @mock.patch.object(report.Report, '_get_metrics')
     @mock.patch.object(report.Report, '_get_fieldkeys')
     def test__generate_common(self, mock_keys, mock_metrics):
@@ -248,12 +550,33 @@ class ReportTestCase(unittest.TestCase):
         mock_keys.assert_called_once_with()
         self.assertEqual(GOOD_TIMESTAMP, self.rep.Timestamp)
 
+    @mock.patch.object(report.Report, '_get_baro_metrics')
     @mock.patch.object(report.Report, '_get_metrics')
     @mock.patch.object(report.Report, '_get_fieldkeys')
     @mock.patch.object(report.Report, '_validate')
-    def test_generate_nsb(self, mock_valid, mock_keys, mock_metrics):
+    def test_generate_nsb(
+        self, mock_valid, mock_keys, mock_metrics, mock_baro_metrics):
+
         mock_metrics.return_value = GOOD_DB_METRICS
         mock_keys.return_value = GOOD_DB_FIELDKEYS
+        BARO_METRICS = {
+            # TODO: is timestamp needed here?
+            'Timestamp': [
+                '14:11:25.383698', '14:11:25.383712', '14:11:35.383696',
+                '14:11:35.383713', '14:11:45.383700', '14:11:45.383716'],
+            'myhostname.cpu_value.cpu.user.0': {
+                '14:11:25.383698': 324050,
+                '14:11:35.383696': 324051,
+                '14:11:45.383700': 324054,
+            },
+            'myhostname.cpu_value.cpu.system.0': {
+                '14:11:25.383712': 193798,
+                '14:11:35.383713': 193800,
+                '14:11:45.383716': 193801,
+            }
+        }
+        mock_baro_metrics.return_value = BARO_METRICS
+
         self.rep.generate_nsb(self.param)
         mock_valid.assert_called_once_with(GOOD_YAML_NAME, GOOD_TASK_ID)
         mock_metrics.assert_called_once_with()