From 0738c2544e3220a3e0fd80a77ab1f2800f90533e Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 24 Jan 2019 15:19:44 +0000 Subject: Refactor: add _format_datasets JIRA: YARDSTICK-1593 Change-Id: I8d2c8665d767c92da20db8f97690f20da4a68908 Signed-off-by: Emma Foley --- yardstick/tests/unit/benchmark/core/test_report.py | 60 ++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) (limited to 'yardstick/tests/unit/benchmark') diff --git a/yardstick/tests/unit/benchmark/core/test_report.py b/yardstick/tests/unit/benchmark/core/test_report.py index b498299a9..77e949c66 100644 --- a/yardstick/tests/unit/benchmark/core/test_report.py +++ b/yardstick/tests/unit/benchmark/core/test_report.py @@ -77,10 +77,10 @@ MORE_EXPECTED_TABLE_VALS = { 123, 4.56, 9876543210987654321 if six.PY3 else 9.876543210987655e+18, - 'str_str value', - 'str_unicode value', - 'unicode_str value', - 'unicode_unicode value', + None, + None, + None, + None, 7.89, 1011, 9876543210123456789 if six.PY3 else 9.876543210123457e+18, @@ -227,6 +227,58 @@ class ReportTestCase(unittest.TestCase): self.rep._get_timestamps(metrics) ) + def test__format_datasets(self): + metric_name = "free.memory0.used" + metrics = [{ + u'free.memory1.free': u'1958664', + u'free.memory0.used': u'9789560', + }, { + u'free.memory1.free': u'1958228', + u'free.memory0.used': u'9789790', + }, { + u'free.memory1.free': u'1956156', + u'free.memory0.used': u'9791092', + }, { + u'free.memory1.free': u'1956280', + u'free.memory0.used': u'9790796', + }] + self.assertEqual( + [9789560, 9789790, 9791092, 9790796,], + self.rep._format_datasets(metric_name, metrics) + ) + + def test__format_datasets_val_none(self): + metric_name = "free.memory0.used" + metrics = [{ + u'free.memory1.free': u'1958664', + u'free.memory0.used': 9876543109876543210, + }, { + u'free.memory1.free': u'1958228', + }, { + u'free.memory1.free': u'1956156', + u'free.memory0.used': u'9791092', + }, { + u'free.memory1.free': u'1956280', + u'free.memory0.used': u'9790796', + }] + + exp0 = 9876543109876543210 if six.PY3 else 9.876543109876543e+18 + self.assertEqual( + [exp0, None, 9791092, 9790796], + self.rep._format_datasets(metric_name, metrics) + ) + + def test__format_datasets_val_incompatible(self): + metric_name = "free.memory0.used" + metrics = [{ + u'free.memory0.used': "some incompatible value", + }, { + }] + self.assertEqual( + [None, None], + self.rep._format_datasets(metric_name, metrics) + ) + @mock.patch.object(report.Report, '_get_metrics') @mock.patch.object(report.Report, '_get_fieldkeys') def test__generate_common(self, mock_keys, mock_metrics): -- cgit 1.2.3-korg From 498fcc39a9f948f6911157c5cba27986e3f78208 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 28 Jan 2019 21:05:11 +0000 Subject: benchmark.core.report: Add _get_baro_metrics Retrieve and parse Barometer metrics relevant to a Yardstick testcase. _get_barometer_metrics() retrieves NFVi metrics from collectd for metrics collected during a Yardstick test case. JIRA: YARDSTICK-1593 Change-Id: I1ef1ccd999e1cbee546db39fb8ad6de0d97e6576 Signed-off-by: Emma Foley --- yardstick/benchmark/core/report.py | 60 ++++++++++ yardstick/tests/unit/benchmark/core/test_report.py | 132 ++++++++++++++++++++- 2 files changed, 191 insertions(+), 1 deletion(-) (limited to 'yardstick/tests/unit/benchmark') diff --git a/yardstick/benchmark/core/report.py b/yardstick/benchmark/core/report.py index 30ba243c3..3d5912471 100644 --- a/yardstick/benchmark/core/report.py +++ b/yardstick/benchmark/core/report.py @@ -121,6 +121,66 @@ class Report(object): else: raise KeyError("Task ID or Test case not found.") + def _get_task_start_time(self): + # The start time should come from the task or the metadata table. + # The first entry into influx for a task will be AFTER the first TC + # iteration + cmd = "select * from \"%s\" where task_id='%s' ORDER BY time ASC limit 1" + task_query = cmd % (self.yaml_name, self.task_id) + + query_exec = influx.query(task_query) + start_time = query_exec[0]['time'] + return start_time + + def _get_task_end_time(self): + # NOTE(elfoley): when using select first() and select last() for the + # DB query, the timestamp returned is 0, so later queries try to + # return metrics from 1970 + cmd = "select * from \"%s\" where task_id='%s' ORDER BY time DESC limit 1" + task_query = cmd % (self.yaml_name, self.task_id) + query_exec = influx.query(task_query) + end_time = query_exec[0]['time'] + return end_time + + def _get_baro_metrics(self): + start_time = self._get_task_start_time() + end_time = self._get_task_end_time() + metric_list = [ + "cpu_value", "cpufreq_value", "intel_pmu_value", + "virt_value", "memory_value"] + metrics = {} + times = [] + query_exec = {} + for metric in metric_list: + cmd = "select * from \"%s\" where time >= '%s' and time <= '%s'" + query = cmd % (metric, start_time, end_time) + query_exec[metric] = influx.query(query, db='collectd') + print("query_exec: {}".format(query_exec)) + + for metric in query_exec: + print("metric in query_exec: {}".format(metric)) + met_values = query_exec[metric] + print("met_values: {}".format(met_values)) + for x in met_values: + x['name'] = metric + metric_name = str('.'.join( + [x[f] for f in [ + 'host', 'name', 'type', 'type_instance', 'instance' + ] if x.get(f)])) + + if not metrics.get(metric_name): + metrics[metric_name] = {} + metric_time = self._get_trimmed_timestamp(x['time']) + times.append(metric_time) + time = metric_time + metrics[metric_name][time] = x['value'] + + times = sorted(list(set(times))) + + metrics['Timestamp'] = times + print("metrics: {}".format(metrics)) + return metrics + def _get_trimmed_timestamp(self, metric_time, resolution=4): if not isinstance(metric_time, str): metric_time = metric_time.encode('utf8') # PY2: unicode to str diff --git a/yardstick/tests/unit/benchmark/core/test_report.py b/yardstick/tests/unit/benchmark/core/test_report.py index 77e949c66..a92eda1b8 100644 --- a/yardstick/tests/unit/benchmark/core/test_report.py +++ b/yardstick/tests/unit/benchmark/core/test_report.py @@ -219,6 +219,115 @@ class ReportTestCase(unittest.TestCase): self.rep.task_id = GOOD_TASK_ID six.assertRaisesRegex(self, KeyError, "Task ID", self.rep._get_metrics) + @mock.patch.object(influx, 'query') + def test__get_task_start_time(self, mock_query): + self.rep.yaml_name = GOOD_YAML_NAME + self.rep.task_id = GOOD_TASK_ID + mock_query.return_value = [{ + u'free.memory0.used': u'9789088', + u'free.memory0.available': u'22192984', + u'free.memory0.shared': u'219152', + u'time': u'2019-01-22T16:20:14.568075776Z', + }] + expected = "2019-01-22T16:20:14.568075776Z" + + self.assertEqual( + expected, + self.rep._get_task_start_time() + ) + + def test__get_task_start_time_task_not_found(self): + pass + + @mock.patch.object(influx, 'query') + def test__get_task_end_time(self, mock_query): + self.rep.yaml_name = GOOD_YAML_NAME + self.rep.task_id = GOOD_TASK_ID + # TODO(elfoley): write this test! + mock_query.return_value = [{ + + }] + + @mock.patch.object(influx, 'query') + def test__get_baro_metrics(self, mock_query): + self.rep.yaml_name = GOOD_YAML_NAME + self.rep.task_id = GOOD_TASK_ID + self.rep._get_task_start_time = mock.Mock(return_value=0) + self.rep._get_task_end_time = mock.Mock(return_value=0) + + influx_return_values = ([{ + u'value': 324050, u'instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:25.383698038Z', + u'type_instance': u'user', u'type': u'cpu', + }, { + u'value': 193798, u'instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:25.383712594Z', + u'type_instance': u'system', u'type': u'cpu', + }, { + u'value': 324051, u'instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:35.383696624Z', + u'type_instance': u'user', u'type': u'cpu', + }, { + u'value': 193800, u'instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:35.383713481Z', + u'type_instance': u'system', u'type': u'cpu', + }, { + u'value': 324054, u'instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:45.3836966789Z', + u'type_instance': u'user', u'type': u'cpu', + }, { + u'value': 193801, u'instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:45.383716296Z', + u'type_instance': u'system', u'type': u'cpu', + }], + [{ + u'value': 3598453000, u'host': u'myhostname', + u'time': u'2018-12-19T14:11:25.383698038Z', + u'type_instance': u'0', u'type': u'cpufreq', + }, { + u'value': 3530250000, u'type_instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:35.383712594Z', u'type': u'cpufreq', + }, { + u'value': 3600281000, u'type_instance': u'0', u'host': u'myhostname', + u'time': u'2018-12-19T14:11:45.383696624Z', u'type': u'cpufreq', + }], + ) + + def ret_vals(vals): + for x in vals: + yield x + while True: + yield [] + + mock_query.side_effect = ret_vals(influx_return_values) + + BARO_EXPECTED_METRICS = { + 'Timestamp': [ + '14:11:25.3836', '14:11:25.3837', + '14:11:35.3836', '14:11:35.3837', + '14:11:45.3836', '14:11:45.3837'], + 'myhostname.cpu_value.cpu.user.0': { + '14:11:25.3836': 324050, + '14:11:35.3836': 324051, + '14:11:45.3836': 324054, + }, + 'myhostname.cpu_value.cpu.system.0': { + '14:11:25.3837': 193798, + '14:11:35.3837': 193800, + '14:11:45.3837': 193801, + }, + 'myhostname.cpufreq_value.cpufreq.0': { + '14:11:25.3836': 3598453000, + '14:11:35.3837': 3530250000, + '14:11:45.3836': 3600281000, + } + } + self.maxDiff = None + self.assertEqual( + BARO_EXPECTED_METRICS, + self.rep._get_baro_metrics() + ) + def test__get_timestamps(self): metrics = MORE_DB_METRICS @@ -300,12 +409,33 @@ class ReportTestCase(unittest.TestCase): mock_keys.assert_called_once_with() self.assertEqual(GOOD_TIMESTAMP, self.rep.Timestamp) + @mock.patch.object(report.Report, '_get_baro_metrics') @mock.patch.object(report.Report, '_get_metrics') @mock.patch.object(report.Report, '_get_fieldkeys') @mock.patch.object(report.Report, '_validate') - def test_generate_nsb(self, mock_valid, mock_keys, mock_metrics): + def test_generate_nsb( + self, mock_valid, mock_keys, mock_metrics, mock_baro_metrics): + mock_metrics.return_value = GOOD_DB_METRICS mock_keys.return_value = GOOD_DB_FIELDKEYS + BARO_METRICS = { + # TODO: is timestamp needed here? + 'Timestamp': [ + '14:11:25.383698', '14:11:25.383712', '14:11:35.383696', + '14:11:35.383713', '14:11:45.383700', '14:11:45.383716'], + 'myhostname.cpu_value.cpu.user.0': { + '14:11:25.383698': 324050, + '14:11:35.383696': 324051, + '14:11:45.383700': 324054, + }, + 'myhostname.cpu_value.cpu.system.0': { + '14:11:25.383712': 193798, + '14:11:35.383713': 193800, + '14:11:45.383716': 193801, + } + } + mock_baro_metrics.return_value = BARO_METRICS + self.rep.generate_nsb(self.param) mock_valid.assert_called_once_with(GOOD_YAML_NAME, GOOD_TASK_ID) mock_metrics.assert_called_once_with() -- cgit 1.2.3-korg From 6a0ab66ed2890c7236db8ff49cde909f24f5d92a Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 25 Jan 2019 14:42:55 +0000 Subject: benchmark.core.report: Add _combine_times _combine_times() combines and interleves separate lists of timestamps. This will allow the Yardstick and Barometer timestamps to be combined for the dynamic HTML report created with ``yardstick report generate-nsb`` command. JIRA: YARDSTICK-1593 Change-Id: I0f5ea4d001775495fb9b9b4de2d2360c9c61cc51 Signed-off-by: Emma Foley --- yardstick/benchmark/core/report.py | 31 +++++ yardstick/tests/unit/benchmark/core/test_report.py | 142 +++++++++++++++++++++ 2 files changed, 173 insertions(+) (limited to 'yardstick/tests/unit/benchmark') diff --git a/yardstick/benchmark/core/report.py b/yardstick/benchmark/core/report.py index 3d5912471..587c85a14 100644 --- a/yardstick/benchmark/core/report.py +++ b/yardstick/benchmark/core/report.py @@ -305,6 +305,37 @@ class Report(object): print("Report generated. View %s" % consts.DEFAULT_HTML_FILE) + def _combine_times(self, *args): + times = [] + # Combines an arbitrary number of lists + [times.extend(x) for x in args] + times = list(set(times)) + times.sort() + return times + + def _combine_metrics(self, *args): + baro_data, baro_time, yard_data, yard_time = args + combo_time = self._combine_times(baro_time, yard_time) + + data = {} + [data.update(x) for x in (baro_data, yard_data)] + + table_data = {} + table_data['Timestamp'] = combo_time + combo = {} + keys = sorted(data.keys()) + for met_name in data: + dataset = [] + for point in data[met_name]: + dataset.append({'x': point, 'y': data[met_name][point]}) + # the metrics need to be ordered by time + combo[met_name] = sorted(dataset, key=lambda i: i['x']) + for met_name in data: + table_data[met_name] = [] + for t in combo_time: + table_data[met_name].append(data[met_name].get(t, '')) + return combo, keys, table_data + @cliargs("task_id", type=str, help=" task id", nargs=1) @cliargs("yaml_name", type=str, help=" Yaml file Name", nargs=1) def generate_nsb(self, args): diff --git a/yardstick/tests/unit/benchmark/core/test_report.py b/yardstick/tests/unit/benchmark/core/test_report.py index a92eda1b8..f3b17f30a 100644 --- a/yardstick/tests/unit/benchmark/core/test_report.py +++ b/yardstick/tests/unit/benchmark/core/test_report.py @@ -388,6 +388,148 @@ class ReportTestCase(unittest.TestCase): self.rep._format_datasets(metric_name, metrics) ) + def test__combine_times(self): + yard_times = [ + '00:00:00.000000', + '00:00:01.000000', + '00:00:02.000000', + '00:00:06.000000', + '00:00:08.000000', + '00:00:09.000000', + ] + baro_times = [ + '00:00:01.000000', + '00:00:03.000000', + '00:00:04.000000', + '00:00:05.000000', + '00:00:07.000000', + '00:00:10.000000', + ] + expected_combo = [ + '00:00:00.000000', + '00:00:01.000000', + '00:00:02.000000', + '00:00:03.000000', + '00:00:04.000000', + '00:00:05.000000', + '00:00:06.000000', + '00:00:07.000000', + '00:00:08.000000', + '00:00:09.000000', + '00:00:10.000000', + ] + + actual_combo = self.rep._combine_times(yard_times, baro_times) + self.assertEqual(len(expected_combo), len(actual_combo)) + + self.assertEqual( + expected_combo, + actual_combo, + ) + + def test__combine_times_2(self): + time1 = ['14:11:25.383698', '14:11:25.383712', '14:11:35.383696',] + time2 = [ + '16:20:14.568075', '16:20:24.575083', + '16:20:34.580989', '16:20:44.586801', ] + time_exp = [ + '14:11:25.383698', '14:11:25.383712', '14:11:35.383696', + '16:20:14.568075', '16:20:24.575083', '16:20:34.580989', + '16:20:44.586801', + ] + self.assertEqual(time_exp, self.rep._combine_times(time1, time2)) + + def test__combine_metrics(self): + BARO_METRICS = { + 'myhostname.cpu_value.cpu.user.0': { + '14:11:25.3836': 324050, '14:11:35.3836': 324051, + '14:11:45.3836': 324054, + }, + 'myhostname.cpu_value.cpu.system.0': { + '14:11:25.3837': 193798, '14:11:35.3837': 193800, + '14:11:45.3837': 193801, + } + } + BARO_TIMES = [ + '14:11:25.3836', '14:11:25.3837', '14:11:35.3836', + '14:11:35.3837', '14:11:45.3836', '14:11:45.3837', + ] + YARD_METRICS = { + 'free.memory9.free': { + '16:20:14.5680': 1958244, '16:20:24.5750': 1955964, + '16:20:34.5809': 1956040, '16:20:44.5868': 1956428, + }, + 'free.memory7.used': { + '16:20:14.5680': 9789068, '16:20:24.5750': 9791284, + '16:20:34.5809': 9791228, '16:20:44.5868': 9790692, + }, + 'free.memory2.total':{ + '16:20:14.5680': 32671288, '16:20:24.5750': 32671288, + '16:20:34.5809': 32671288, '16:20:44.5868': 32671288, + }, + 'free.memory7.free': { + '16:20:14.5680': 1958368, '16:20:24.5750': 1956104, + '16:20:34.5809': 1956040, '16:20:44.5868': 1956552, + }, + 'free.memory1.used': { + '16:20:14.5680': 9788872, '16:20:24.5750': 9789212, + '16:20:34.5809': 9791168, '16:20:44.5868': 9790996, + }, + } + YARD_TIMES = [ + '16:20:14.5680', '16:20:24.5750', + '16:20:34.5809', '16:20:44.5868', + ] + + expected_output = { + 'myhostname.cpu_value.cpu.user.0': [{ + 'x': '14:11:25.3836', 'y': 324050, }, { + 'x': '14:11:35.3836', 'y': 324051, }, { + 'x': '14:11:45.3836', 'y': 324054, }], + 'myhostname.cpu_value.cpu.system.0' : [{ + 'x': '14:11:25.3837', 'y': 193798, }, { + 'x': '14:11:35.3837', 'y': 193800, }, { + 'x': '14:11:45.3837', 'y': 193801, }], + 'free.memory9.free': [{ + 'x': '16:20:14.5680', 'y': 1958244, }, { + 'x': '16:20:24.5750', 'y': 1955964, }, { + 'x': '16:20:34.5809', 'y': 1956040, }, { + 'x': '16:20:44.5868', 'y': 1956428, }], + 'free.memory7.used': [{ + 'x': '16:20:14.5680', 'y': 9789068, }, { + 'x': '16:20:24.5750', 'y': 9791284, }, { + 'x': '16:20:34.5809', 'y': 9791228, }, { + 'x': '16:20:44.5868', 'y': 9790692, }], + 'free.memory2.total': [{ + 'x': '16:20:14.5680', 'y': 32671288, }, { + 'x': '16:20:24.5750', 'y': 32671288, }, { + 'x': '16:20:34.5809', 'y': 32671288, }, { + 'x': '16:20:44.5868', 'y': 32671288, }], + 'free.memory7.free': [{ + 'x': '16:20:14.5680', 'y': 1958368, }, { + 'x': '16:20:24.5750', 'y': 1956104, }, { + 'x': '16:20:34.5809', 'y': 1956040, }, { + 'x': '16:20:44.5868', 'y': 1956552, }], + 'free.memory1.used': [{ + 'x': '16:20:14.5680', 'y': 9788872, }, { + 'x': '16:20:24.5750', 'y': 9789212, }, { + 'x': '16:20:34.5809', 'y': 9791168, }, { + 'x': '16:20:44.5868', 'y': 9790996, }], + } + + actual_output, _, _ = self.rep._combine_metrics( + BARO_METRICS, BARO_TIMES, YARD_METRICS, YARD_TIMES + ) + self.assertEquals( + sorted(expected_output.keys()), + sorted(actual_output.keys()) + ) + + self.assertEquals( + expected_output, + actual_output, + ) + @mock.patch.object(report.Report, '_get_metrics') @mock.patch.object(report.Report, '_get_fieldkeys') def test__generate_common(self, mock_keys, mock_metrics): -- cgit 1.2.3-korg From 34276b464ac7a2013d292680e545d25e515bb0cd Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 25 Jan 2019 14:44:20 +0000 Subject: Use baro and yardstick metrics in dynamic HTML report _combine_metrics combines metrics from different sources. This is for use with the ``yardstick report generate-nsb`` command, which will combine yardstick and barometer metrics in the dynamic HTML report. JIRA: YARDSTICK-1593 Change-Id: I87002948ebb4cc88fb0932380bcb9920eb53db58 Signed-off-by: Emma Foley --- yardstick/benchmark/core/report.py | 30 ++- yardstick/common/nsb_report.html.j2 | 7 +- yardstick/common/nsb_report.js | 11 +- .../tests/functional/benchmark/core/test_report.py | 226 +++++++++++++++++++-- yardstick/tests/unit/benchmark/core/test_report.py | 1 - 5 files changed, 249 insertions(+), 26 deletions(-) (limited to 'yardstick/tests/unit/benchmark') diff --git a/yardstick/benchmark/core/report.py b/yardstick/benchmark/core/report.py index 587c85a14..e5dc62050 100644 --- a/yardstick/benchmark/core/report.py +++ b/yardstick/benchmark/core/report.py @@ -342,25 +342,43 @@ class Report(object): """Start NSB report generation.""" _, report_data = self._generate_common(args) report_time = report_data.pop('Timestamp') - report_keys = sorted(report_data, key=str.lower) - report_tree = JSTree().format_for_jstree(report_keys) report_meta = { "testcase": self.yaml_name, "task_id": self.task_id, } + yardstick_data = {} + for i, t in enumerate(report_time): + for m in report_data: + if not yardstick_data.get(m): + yardstick_data[m] = {} + yardstick_data[m][t] = report_data[m][i] + + baro_data = self._get_baro_metrics() + baro_timestamps = baro_data.pop('Timestamp') + + yard_timestamps = report_time + report_time = self._combine_times(yard_timestamps, baro_timestamps) + + combo_metrics, combo_keys, combo_table = self._combine_metrics( + baro_data, baro_timestamps, yardstick_data, yard_timestamps) + combo_time = self._combine_times(baro_timestamps, yard_timestamps) + combo_tree = JSTree().format_for_jstree(combo_keys) + template_dir = consts.YARDSTICK_ROOT_PATH + "yardstick/common" template_environment = jinja2.Environment( autoescape=False, loader=jinja2.FileSystemLoader(template_dir), lstrip_blocks=True) + combo_data = combo_metrics context = { "report_meta": report_meta, - "report_data": report_data, - "report_time": report_time, - "report_keys": report_keys, - "report_tree": report_tree, + "report_data": combo_data, + "report_time": combo_time, + "report_keys": combo_keys, + "report_tree": combo_tree, + "table_data": combo_table, } template_html = template_environment.get_template("nsb_report.html.j2") diff --git a/yardstick/common/nsb_report.html.j2 b/yardstick/common/nsb_report.html.j2 index aa90253f8..a6713eb16 100644 --- a/yardstick/common/nsb_report.html.j2 +++ b/yardstick/common/nsb_report.html.j2 @@ -3,7 +3,7 @@