Fix conversion to JS for HTML reports

Some new fields in InfluxDB have plain text contents that AST cannot
parse as valid expressions, resulting in exceptions being raised by
yardstick CLI commands "report generate" and "report generate-nsb".

Reworked _generate_common() to properly handle and convert:
- unicode, str and missing keys
- None, unicode, str, float, long and int values
- float, long and int stored as unicode or str values

Added test__generate_common() unit test, to verify all possible
conversion flavors.

Also renamed incorrect "tasks" to "metrics".

Also fixed warning in Python3 for some unit tests, caused by
deprecation of unittest.TestCase.assertRaisesRegexp().

JIRA: YARDSTICK-1367
Topic: report/html_table (10 of 12)

Change-Id: Iff75bfd2c1dcaf3806f67f52f0ea594f10aceb5b
Signed-off-by: Patrice Buriez <patrice.buriez@intel.com>

1 files changed, 96 insertions, 17 deletions

diff --git a/yardstick/tests/unit/benchmark/core/test_report.py b/yardstick/tests/unit/benchmark/core/test_report.py
index 11d017ff0..41991ddd4 100644
--- a/yardstick/tests/unit/benchmark/core/test_report.py
+++ b/yardstick/tests/unit/benchmark/core/test_report.py
@@ -9,6 +9,7 @@
 ##############################################################################
 
 import mock
+import six
 import unittest
 import uuid
 
@@ -19,13 +20,82 @@ from yardstick.cmd.commands import change_osloobj_to_paras
 GOOD_YAML_NAME = 'fake_name'
 GOOD_TASK_ID = str(uuid.uuid4())
 GOOD_DB_FIELDKEYS = [{'fieldKey': 'fake_key'}]
-GOOD_DB_TASK = [{
+GOOD_DB_METRICS = [{
         'fake_key': 1.234,
         'time': '0000-00-00T12:34:56.789012Z',
         }]
 GOOD_TIMESTAMP = ['12:34:56.789012']
 BAD_YAML_NAME = 'F@KE_NAME'
 BAD_TASK_ID = 'aaaaaa-aaaaaaaa-aaaaaaaaaa-aaaaaa'
+MORE_DB_FIELDKEYS = [
+        {'fieldKey': 'fake_key'},
+        {'fieldKey': 'str_str'},
+        {'fieldKey': u'str_unicode'},
+        {u'fieldKey': 'unicode_str'},
+        {u'fieldKey': u'unicode_unicode'},
+        ]
+MORE_DB_METRICS = [{
+        'fake_key': None,
+        'time': '0000-00-00T00:00:00.000000Z',
+        }, {
+        'fake_key': 123,
+        'time': '0000-00-00T00:00:01.000000Z',
+        }, {
+        'fake_key': 4.56,
+        'time': '0000-00-00T00:00:02.000000Z',
+        }, {
+        'fake_key': 9876543210987654321,
+        'time': '0000-00-00T00:00:03.000000Z',
+        }, {
+        'fake_key': 'str_str value',
+        'time': '0000-00-00T00:00:04.000000Z',
+        }, {
+        'fake_key': u'str_unicode value',
+        'time': '0000-00-00T00:00:05.000000Z',
+        }, {
+        u'fake_key': 'unicode_str value',
+        'time': '0000-00-00T00:00:06.000000Z',
+        }, {
+        u'fake_key': u'unicode_unicode value',
+        'time': '0000-00-00T00:00:07.000000Z',
+        }, {
+        'fake_key': '7.89',
+        'time': '0000-00-00T00:00:08.000000Z',
+        }, {
+        'fake_key': '1011',
+        'time': '0000-00-00T00:00:09.000000Z',
+        }, {
+        'fake_key': '9876543210123456789',
+        'time': '0000-00-00T00:00:10.000000Z',
+        }]
+MORE_TIMESTAMP = ['00:00:%02d.000000' % n for n in range(len(MORE_DB_METRICS))]
+MORE_EMPTY_DATA = [None] * len(MORE_DB_METRICS)
+MORE_EXPECTED_TABLE_VALS = {
+        'Timestamp': MORE_TIMESTAMP,
+        'fake_key': [
+            None,
+            123,
+            4.56,
+            9876543210987654321 if six.PY3 else 9.876543210987655e+18,
+            'str_str value',
+            'str_unicode value',
+            'unicode_str value',
+            'unicode_unicode value',
+            7.89,
+            1011,
+            9876543210123456789 if six.PY3 else 9.876543210123457e+18,
+            ],
+        'str_str': MORE_EMPTY_DATA,
+        'str_unicode': MORE_EMPTY_DATA,
+        'unicode_str': MORE_EMPTY_DATA,
+        'unicode_unicode': MORE_EMPTY_DATA,
+        }
+MORE_EXPECTED_DATASETS = [{
+        'label': key,
+        'data': MORE_EXPECTED_TABLE_VALS[key],
+        }
+        for key in map(str, [field['fieldKey'] for field in MORE_DB_FIELDKEYS])
+        ]
 
 
 class JSTreeTestCase(unittest.TestCase):
@@ -117,11 +187,11 @@ class ReportTestCase(unittest.TestCase):
         self.assertEqual(GOOD_TASK_ID, str(self.rep.task_id))
 
     def test__validate_invalid_yaml_name(self):
-        with self.assertRaisesRegexp(ValueError, "yaml*"):
+        with six.assertRaisesRegex(self, ValueError, "yaml*"):
             self.rep._validate(BAD_YAML_NAME, GOOD_TASK_ID)
 
     def test__validate_invalid_task_id(self):
-        with self.assertRaisesRegexp(ValueError, "task*"):
+        with six.assertRaisesRegex(self, ValueError, "task*"):
             self.rep._validate(GOOD_YAML_NAME, BAD_TASK_ID)
 
     @mock.patch.object(influx, 'query')
@@ -141,42 +211,51 @@ class ReportTestCase(unittest.TestCase):
         mock_query.return_value = []
         self.rep.yaml_name = GOOD_YAML_NAME
         self.rep.task_id = GOOD_TASK_ID
-        self.assertRaisesRegexp(KeyError, "Test case", self.rep._get_fieldkeys)
+        six.assertRaisesRegex(self, KeyError, "Test case", self.rep._get_fieldkeys)
 
     @mock.patch.object(influx, 'query')
-    def test__get_tasks(self, mock_query):
-        mock_query.return_value = GOOD_DB_TASK
+    def test__get_metrics(self, mock_query):
+        mock_query.return_value = GOOD_DB_METRICS
         self.rep.yaml_name = GOOD_YAML_NAME
         self.rep.task_id = GOOD_TASK_ID
-        self.assertEqual(GOOD_DB_TASK, self.rep._get_tasks())
+        self.assertEqual(GOOD_DB_METRICS, self.rep._get_metrics())
 
     @mock.patch.object(influx, 'query')
-    def test__get_tasks_task_not_found(self, mock_query):
+    def test__get_metrics_task_not_found(self, mock_query):
         mock_query.return_value = []
         self.rep.yaml_name = GOOD_YAML_NAME
         self.rep.task_id = GOOD_TASK_ID
-        self.assertRaisesRegexp(KeyError, "Task ID", self.rep._get_tasks)
+        six.assertRaisesRegex(self, KeyError, "Task ID", self.rep._get_metrics)
+
+    @mock.patch.object(report.Report, '_get_metrics')
+    @mock.patch.object(report.Report, '_get_fieldkeys')
+    def test__generate_common(self, mock_keys, mock_metrics):
+        mock_metrics.return_value = MORE_DB_METRICS
+        mock_keys.return_value = MORE_DB_FIELDKEYS
+        datasets, table_vals = self.rep._generate_common(self.param)
+        self.assertEqual(MORE_EXPECTED_DATASETS, datasets)
+        self.assertEqual(MORE_EXPECTED_TABLE_VALS, table_vals)
 
-    @mock.patch.object(report.Report, '_get_tasks')
+    @mock.patch.object(report.Report, '_get_metrics')
     @mock.patch.object(report.Report, '_get_fieldkeys')
     @mock.patch.object(report.Report, '_validate')
-    def test_generate(self, mock_valid, mock_keys, mock_tasks):
-        mock_tasks.return_value = GOOD_DB_TASK
+    def test_generate(self, mock_valid, mock_keys, mock_metrics):
+        mock_metrics.return_value = GOOD_DB_METRICS
         mock_keys.return_value = GOOD_DB_FIELDKEYS
         self.rep.generate(self.param)
         mock_valid.assert_called_once_with(GOOD_YAML_NAME, GOOD_TASK_ID)
-        mock_tasks.assert_called_once_with()
+        mock_metrics.assert_called_once_with()
         mock_keys.assert_called_once_with()
         self.assertEqual(GOOD_TIMESTAMP, self.rep.Timestamp)
 
-    @mock.patch.object(report.Report, '_get_tasks')
+    @mock.patch.object(report.Report, '_get_metrics')
     @mock.patch.object(report.Report, '_get_fieldkeys')
     @mock.patch.object(report.Report, '_validate')
-    def test_generate_nsb(self, mock_valid, mock_keys, mock_tasks):
-        mock_tasks.return_value = GOOD_DB_TASK
+    def test_generate_nsb(self, mock_valid, mock_keys, mock_metrics):
+        mock_metrics.return_value = GOOD_DB_METRICS
         mock_keys.return_value = GOOD_DB_FIELDKEYS
         self.rep.generate_nsb(self.param)
         mock_valid.assert_called_once_with(GOOD_YAML_NAME, GOOD_TASK_ID)
-        mock_tasks.assert_called_once_with()
+        mock_metrics.assert_called_once_with()
         mock_keys.assert_called_once_with()
         self.assertEqual(GOOD_TIMESTAMP, self.rep.Timestamp)