Disk Initialization and Custom Workload

Removes the deprecated nowarm and nosdd parameters from the
call to StorPerf.

Adds a call to initialize the disk right after the stack
creation is complete so that the results from the
performance run are valid and not skewed by a disk that
is not operating at a consistent level of performance.

Adds the jobs API v2.0 when the custom parameter 'workloads'
is used instead of workload.

Improves documentation and adds more info about the
workload and workloads parameters.

Co-Authored-By: Ameed Ashour <Ameed.Ashour.Ext@nokia.com>

Change-Id: I17e96028f6a82e2c800c8f73c7be866e1cab6ad8
JIRA: YARDSTICK-1322
Signed-off-by: mbeierl <mark.beierl@dell.com>

1 files changed, 310 insertions, 30 deletions

diff --git a/yardstick/tests/unit/benchmark/scenarios/storage/test_storperf.py b/yardstick/tests/unit/benchmark/scenarios/storage/test_storperf.py
index 5844746ab..2ba53cb93 100644
--- a/yardstick/tests/unit/benchmark/scenarios/storage/test_storperf.py
+++ b/yardstick/tests/unit/benchmark/scenarios/storage/test_storperf.py
@@ -11,18 +11,18 @@
 
 from __future__ import absolute_import
 
+import json
 import unittest
 
 import mock
 from oslo_serialization import jsonutils
+import requests
 
 from yardstick.benchmark.scenarios.storage import storperf
 
 
 # pylint: disable=unused-argument
 # disable this for now because I keep forgetting mock patch arg ordering
-
-
 def mocked_requests_config_post(*args, **kwargs):
     class MockResponseConfigPost(object):
 
@@ -32,10 +32,24 @@ def mocked_requests_config_post(*args, **kwargs):
 
     return MockResponseConfigPost(
         '{"stack_id": "dac27db1-3502-4300-b301-91c64e6a1622",'
-        '"stack_created": "false"}',
+        '"stack_created": false}',
         200)
 
 
+def mocked_requests_config_post_fail(*args, **kwargs):
+    class MockResponseConfigPost(object):
+
+        def __init__(self, json_data, status_code):
+            self.content = json_data
+            self.status_code = status_code
+
+    return MockResponseConfigPost(
+        '{"message": "ERROR: Parameter \'public_network\' is invalid: ' +
+        'Error validating value \'foo\': Unable to find network with ' +
+        'name or id \'foo\'"}',
+        400)
+
+
 def mocked_requests_config_get(*args, **kwargs):
     class MockResponseConfigGet(object):
 
@@ -45,10 +59,47 @@ def mocked_requests_config_get(*args, **kwargs):
 
     return MockResponseConfigGet(
         '{"stack_id": "dac27db1-3502-4300-b301-91c64e6a1622",'
-        '"stack_created": "true"}',
+        '"stack_created": true}',
         200)
 
 
+def mocked_requests_config_get_not_created(*args, **kwargs):
+    class MockResponseConfigGet(object):
+
+        def __init__(self, json_data, status_code):
+            self.content = json_data
+            self.status_code = status_code
+
+    return MockResponseConfigGet(
+        '{"stack_id": "",'
+        '"stack_created": false}',
+        200)
+
+
+def mocked_requests_config_get_no_payload(*args, **kwargs):
+    class MockResponseConfigGet(object):
+
+        def __init__(self, json_data, status_code):
+            self.content = json_data
+            self.status_code = status_code
+
+    return MockResponseConfigGet(
+        '{}',
+        200)
+
+
+def mocked_requests_initialize_post_fail(*args, **kwargs):
+    class MockResponseJobPost(object):
+
+        def __init__(self, json_data, status_code):
+            self.content = json_data
+            self.status_code = status_code
+
+    return MockResponseJobPost(
+        '{"message": "ERROR: Stack StorPerfAgentGroup does not exist"}',
+        400)
+
+
 def mocked_requests_job_get(*args, **kwargs):
     class MockResponseJobGet(object):
 
@@ -73,6 +124,18 @@ def mocked_requests_job_post(*args, **kwargs):
                                  "d46bfb8c-36f4-4a40-813b-c4b4a437f728"}', 200)
 
 
+def mocked_requests_job_post_fail(*args, **kwargs):
+    class MockResponseJobPost(object):
+
+        def __init__(self, json_data, status_code):
+            self.content = json_data
+            self.status_code = status_code
+
+    return MockResponseJobPost(
+        '{"message": "ERROR: Stack StorPerfAgentGroup does not exist"}',
+        400)
+
+
 def mocked_requests_job_delete(*args, **kwargs):
     class MockResponseJobDelete(object):
 
@@ -100,10 +163,7 @@ def mocked_requests_delete_failed(*args, **kwargs):
             self.json_data = json_data
             self.status_code = status_code
 
-    if args[0] == "http://172.16.0.137:5000/api/v1.0/configurations":
-        return MockResponseDeleteFailed('{"message": "Teardown failed"}', 400)
-
-    return MockResponseDeleteFailed('{}', 404)
+    return MockResponseDeleteFailed('{"message": "Teardown failed"}', 400)
 
 
 class StorPerfTestCase(unittest.TestCase):
@@ -119,11 +179,14 @@ class StorPerfTestCase(unittest.TestCase):
 
         self.result = {}
 
-    @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.post',
-                side_effect=mocked_requests_config_post)
-    @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.get',
-                side_effect=mocked_requests_config_get)
-    def test_successful_setup(self, mock_post, mock_get):
+    @mock.patch.object(requests, 'post')
+    @mock.patch.object(requests, 'get')
+    def test_setup(self, mock_get, mock_post):
+        mock_post.side_effect = [mocked_requests_config_post(),
+                                 mocked_requests_job_post()]
+        mock_get.side_effect = [mocked_requests_config_get(),
+                                mocked_requests_job_get()]
+
         options = {
             "agent_count": 8,
             "public_network": 'ext-net',
@@ -146,14 +209,47 @@ class StorPerfTestCase(unittest.TestCase):
 
         self.assertTrue(s.setup_done)
 
-    @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.post',
-                side_effect=mocked_requests_job_post)
-    @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.get',
-                side_effect=mocked_requests_job_get)
-    @mock.patch(
-        'yardstick.benchmark.scenarios.storage.storperf.requests.delete',
-        side_effect=mocked_requests_job_delete)
-    def test_successful_run(self, mock_post, mock_get, mock_delete):
+    @mock.patch.object(requests, 'get')
+    def test_query_setup_state_unsuccessful(self, mock_get):
+        mock_get.side_effect = mocked_requests_config_get_not_created
+        args = {
+            "options": {}
+        }
+        s = storperf.StorPerf(args, self.ctx)
+        result = s._query_setup_state()
+        self.assertFalse(result)
+
+    @mock.patch.object(requests, 'get')
+    def test_query_setup_state_no_payload(self, mock_get):
+        mock_get.side_effect = mocked_requests_config_get_no_payload
+        args = {
+            "options": {}
+        }
+        s = storperf.StorPerf(args, self.ctx)
+        result = s._query_setup_state()
+        self.assertFalse(result)
+
+    @mock.patch.object(requests, 'post')
+    @mock.patch.object(requests, 'get')
+    def test_setup_config_post_failed(self, mock_get, mock_post):
+        mock_post.side_effect = mocked_requests_config_post_fail
+
+        args = {
+            "options": {
+                "public_network": "foo"
+            }
+        }
+
+        s = storperf.StorPerf(args, self.ctx)
+
+        self.assertRaises(RuntimeError, s.setup)
+
+    @mock.patch.object(requests, 'get')
+    @mock.patch.object(requests, 'post')
+    def test_run_v1_successful(self, mock_post, mock_get):
+        mock_post.side_effect = mocked_requests_job_post
+        mock_get.side_effect = mocked_requests_job_get
+
         options = {
             "agent_count": 8,
             "public_network": 'ext-net',
@@ -165,6 +261,74 @@ class StorPerfTestCase(unittest.TestCase):
             "query_interval": 0,
             "timeout": 60
         }
+        expected_post = {
+            'metadata': {
+                'build_tag': 'latest',
+                'test_case': 'opnfv_yardstick_tc074'
+            },
+            'deadline': 60,
+            'block_sizes': 4096,
+            'queue_depths': 4,
+            "workload": "rs",
+            'agent_count': 8
+        }
+
+        args = {
+            "options": options
+        }
+
+        s = storperf.StorPerf(args, self.ctx)
+        s.setup_done = True
+
+        sample_output = '{"Status": "Completed",\
+         "_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}'
+
+        expected_result = jsonutils.loads(sample_output)
+
+        s.run(self.result)
+
+        mock_post.assert_called_once_with(
+            'http://192.168.23.2:5000/api/v1.0/jobs',
+            json=jsonutils.loads(json.dumps(expected_post)))
+
+        self.assertEqual(self.result, expected_result)
+
+    @mock.patch.object(requests, 'get')
+    @mock.patch.object(requests, 'post')
+    def test_run_v2_successful(self, mock_post, mock_get):
+        mock_post.side_effect = mocked_requests_job_post
+        mock_get.side_effect = mocked_requests_job_get
+
+        options = {
+            "agent_count": 8,
+            "public_network": 'ext-net',
+            "volume_size": 10,
+            "block_sizes": 4096,
+            "queue_depths": 4,
+            "workloads": {
+                "read_sequential": {
+                    "rw": "rs"
+                }
+            },
+            "StorPerf_ip": "192.168.23.2",
+            "query_interval": 0,
+            "timeout": 60
+        }
+        expected_post = {
+            'metadata': {
+                'build_tag': 'latest',
+                'test_case': 'opnfv_yardstick_tc074'
+            },
+            'deadline': 60,
+            'block_sizes': 4096,
+            'queue_depths': 4,
+            'workloads': {
+                'read_sequential': {
+                    'rw': 'rs'
+                }
+            },
+            'agent_count': 8
+        }
 
         args = {
             "options": options
@@ -179,13 +343,126 @@ class StorPerfTestCase(unittest.TestCase):
         expected_result = jsonutils.loads(sample_output)
 
         s.run(self.result)
+        mock_post.assert_called_once_with(
+            'http://192.168.23.2:5000/api/v2.0/jobs',
+            json=expected_post)
 
         self.assertEqual(self.result, expected_result)
 
-    @mock.patch(
-        'yardstick.benchmark.scenarios.storage.storperf.requests.delete',
-        side_effect=mocked_requests_delete)
-    def test_successful_teardown(self, mock_delete):
+    @mock.patch('time.sleep')
+    @mock.patch.object(requests, 'get')
+    @mock.patch.object(requests, 'post')
+    def test_run_failed(self, mock_post, mock_get, _):
+        mock_post.side_effect = mocked_requests_job_post_fail
+        mock_get.side_effect = mocked_requests_job_get
+
+        options = {
+            "agent_count": 8,
+            "public_network": 'ext-net',
+            "volume_size": 10,
+            "block_sizes": 4096,
+            "queue_depths": 4,
+            "workloads": {
+                "read_sequential": {
+                    "rw": "rs"
+                }
+            },
+            "StorPerf_ip": "192.168.23.2",
+            "query_interval": 0,
+            "timeout": 60
+        }
+        expected_post = {
+            'metadata': {
+                'build_tag': 'latest',
+                'test_case': 'opnfv_yardstick_tc074'
+            },
+            'deadline': 60,
+            'block_sizes': 4096,
+            'queue_depths': 4,
+            'workloads': {
+                'read_sequential': {
+                    'rw': 'rs'
+                }
+            },
+            'agent_count': 8
+        }
+
+        args = {
+            "options": options
+        }
+
+        s = storperf.StorPerf(args, self.ctx)
+        s.setup_done = True
+
+        self.assertRaises(RuntimeError, s.run, self.ctx)
+        mock_post.assert_called_once_with(
+            'http://192.168.23.2:5000/api/v2.0/jobs',
+            json=expected_post)
+
+    @mock.patch('time.sleep')
+    @mock.patch.object(requests, 'get')
+    @mock.patch.object(requests, 'post')
+    @mock.patch.object(storperf.StorPerf, 'setup')
+    def test_run_calls_setup(self, mock_setup, mock_post, mock_get, _):
+        mock_post.side_effect = mocked_requests_job_post
+        mock_get.side_effect = mocked_requests_job_get
+
+        args = {
+            "options": {
+                'timeout': 60,
+            }
+        }
+
+        s = storperf.StorPerf(args, self.ctx)
+
+        s.run(self.result)
+
+        mock_setup.assert_called_once()
+
+    @mock.patch('time.sleep')
+    @mock.patch.object(requests, 'get')
+    @mock.patch.object(requests, 'post')
+    def test_initialize_disks(self, mock_post, mock_get, _):
+        mock_post.side_effect = mocked_requests_job_post
+        mock_get.side_effect = mocked_requests_job_get
+
+        args = {
+            "options": {
+                "StorPerf_ip": "192.168.23.2"
+            }
+        }
+
+        s = storperf.StorPerf(args, self.ctx)
+
+        s.initialize_disks()
+
+        mock_post.assert_called_once_with(
+            'http://192.168.23.2:5000/api/v1.0/initializations',
+            json={})
+
+    @mock.patch('time.sleep')
+    @mock.patch.object(requests, 'get')
+    @mock.patch.object(requests, 'post')
+    def test_initialize_disks_post_failed(self, mock_post, mock_get, _):
+        mock_post.side_effect = mocked_requests_initialize_post_fail
+        mock_get.side_effect = mocked_requests_job_get
+
+        args = {
+            "options": {
+                "StorPerf_ip": "192.168.23.2"
+            }
+        }
+
+        s = storperf.StorPerf(args, self.ctx)
+
+        self.assertRaises(RuntimeError, s.initialize_disks)
+        mock_post.assert_called_once_with(
+            'http://192.168.23.2:5000/api/v1.0/initializations',
+            json={})
+
+    @mock.patch.object(requests, 'delete')
+    def test_teardown(self, mock_delete):
+        mock_delete.side_effect = mocked_requests_job_delete
         options = {
             "agent_count": 8,
             "public_network": 'ext-net',
@@ -207,11 +484,12 @@ class StorPerfTestCase(unittest.TestCase):
         s.teardown()
 
         self.assertFalse(s.setup_done)
+        mock_delete.assert_called_once_with(
+            'http://192.168.23.2:5000/api/v1.0/configurations')
 
-    @mock.patch(
-        'yardstick.benchmark.scenarios.storage.storperf.requests.delete',
-        side_effect=mocked_requests_delete_failed)
-    def test_failed_teardown(self, mock_delete):
+    @mock.patch.object(requests, 'delete')
+    def test_teardown_request_delete_failed(self, mock_delete):
+        mock_delete.side_effect = mocked_requests_delete_failed
         options = {
             "agent_count": 8,
             "public_network": 'ext-net',
@@ -230,4 +508,6 @@ class StorPerfTestCase(unittest.TestCase):
 
         s = storperf.StorPerf(args, self.ctx)
 
-        self.assertRaises(AssertionError, s.teardown(), self.result)
+        self.assertRaises(RuntimeError, s.teardown)
+        mock_delete.assert_called_once_with(
+            'http://192.168.23.2:5000/api/v1.0/configurations')