From 8962423227e13f45ef9f7dfbf48ed4847d3f3865 Mon Sep 17 00:00:00 2001 From: mbeierl Date: Fri, 4 May 2018 14:04:51 -0400 Subject: Adds Volume Count and Dynamic Reload Adds a volume_count parameter that allows the user to change the number of volumes. Test execution still uses each volume specified and appends it to the host ip address in the graphite metrics. Changes the cache policy of the stack from SNAPS-OO so that changes made to the stack are refelcted in StorPerf. Fixes the documentation path and updates the docs for the above changes. Change-Id: I5b9160b469e80d4f7d6906ac270bb3b75411fb5a JIRA: STORPERF-242 Signed-off-by: mbeierl (cherry picked from commit 1942e374fba718d1d4b6bc5388803c75e71aa197) --- docker/storperf-master/rest_server.py | 6 + .../storperf/resources/hot/agent-group.yaml | 7 + .../storperf/resources/hot/storperf-agent.yaml | 24 ++- .../storperf/resources/hot/storperf-volume.yaml | 33 ++++ docker/storperf-master/storperf/storperf_master.py | 40 +++- docker/storperf-master/storperf/test_executor.py | 42 +++-- .../storperf/workloads/_base_workload.py | 4 +- .../tests/workload_tests/workload_subclass_test.py | 20 +- docs/dev/gerrit.rst | 203 --------------------- docs/dev/ide.rst | 110 ----------- docs/dev/index.rst | 18 -- docs/dev/initial.rst | 42 ----- docs/dev/unit_tests.rst | 88 --------- docs/release/release-notes/release-notes.rst | 64 +++++-- docs/testing/developer/devguide/gerrit.rst | 203 +++++++++++++++++++++ docs/testing/developer/devguide/ide.rst | 110 +++++++++++ docs/testing/developer/devguide/index.rst | 18 ++ docs/testing/developer/devguide/initial.rst | 42 +++++ docs/testing/developer/devguide/unit_tests.rst | 88 +++++++++ docs/testing/user/installation.rst | 138 +++++++++----- docs/testing/user/test-usage.rst | 46 ++++- 21 files changed, 771 insertions(+), 575 deletions(-) create mode 100644 docker/storperf-master/storperf/resources/hot/storperf-volume.yaml delete mode 100644 docs/dev/gerrit.rst delete mode 100644 docs/dev/ide.rst delete mode 100644 docs/dev/index.rst delete mode 100644 docs/dev/initial.rst delete mode 100644 docs/dev/unit_tests.rst create mode 100644 docs/testing/developer/devguide/gerrit.rst create mode 100644 docs/testing/developer/devguide/ide.rst create mode 100644 docs/testing/developer/devguide/index.rst create mode 100644 docs/testing/developer/devguide/initial.rst create mode 100644 docs/testing/developer/devguide/unit_tests.rst diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py index 7f38ab6..1b66af3 100644 --- a/docker/storperf-master/rest_server.py +++ b/docker/storperf-master/rest_server.py @@ -98,6 +98,7 @@ class ConfigurationRequestModel: 'agent_flavor': fields.String, 'agent_image': fields.String, 'public_network': fields.String, + 'volume_count': fields.Integer, 'volume_size': fields.Integer, 'availability_zone': fields.String, 'username': fields.String, @@ -114,6 +115,7 @@ class ConfigurationResponseModel: 'public_network': fields.String, 'stack_created': fields.Boolean, 'stack_id': fields.String, + 'volume_count': fields.Integer, 'volume_size': fields.Integer, 'availability_zone': fields.String } @@ -135,6 +137,7 @@ class Configure(Resource): 'agent_flavor': storperf.agent_flavor, 'agent_image': storperf.agent_image, 'public_network': storperf.public_network, + 'volume_count': storperf.volume_count, 'volume_size': storperf.volume_size, 'stack_created': storperf.is_stack_created, 'availability_zone': storperf.availability_zone, @@ -170,6 +173,8 @@ class Configure(Resource): storperf.agent_image = request.json['agent_image'] if ('public_network' in request.json): storperf.public_network = request.json['public_network'] + if ('volume_count' in request.json): + storperf.volume_count = request.json['volume_count'] if ('volume_size' in request.json): storperf.volume_size = request.json['volume_size'] if ('availability_zone' in request.json): @@ -188,6 +193,7 @@ class Configure(Resource): 'agent_image': storperf.agent_image, 'availability_zone': storperf.availability_zone, 'public_network': storperf.public_network, + 'volume_count': storperf.volume_count, 'volume_size': storperf.volume_size, 'stack_id': storperf.stack_id}) diff --git a/docker/storperf-master/storperf/resources/hot/agent-group.yaml b/docker/storperf-master/storperf/resources/hot/agent-group.yaml index 3c02e31..ea7b51f 100644 --- a/docker/storperf-master/storperf/resources/hot/agent-group.yaml +++ b/docker/storperf-master/storperf/resources/hot/agent-group.yaml @@ -27,6 +27,12 @@ parameters: constraints: - range: { min: 1, max: 1024 } description: must be between 1 and 1024 Gb. + volume_count: + type: number + default: 0 + constraints: + - range: { min: 0, max: 512 } + description: must be between 1 and 512 agents. agent_count: type: number default: 1 @@ -54,6 +60,7 @@ resources: availability_zone: {get_param: availability_zone}, storperf_open_security_group: {get_resource: storperf_open_security_group}, key_name: {get_resource: storperf_key_pair}, + volume_count: {get_param: volume_count}, volume_size: {get_param: volume_size} } } diff --git a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml index 7841e8c..8895c9f 100644 --- a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml +++ b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml @@ -24,6 +24,13 @@ parameters: default: storperf storperf_open_security_group: type: string + volume_count: + type: number + description: Number of volumes to be created + default: 1 + constraints: + - range: { min: 0, max: 1024 } + description: must be between 1 and 1024. volume_size: type: number description: Size of the volume to be created. @@ -87,15 +94,16 @@ resources: port_id: { get_resource: storperf_agent_port } agent_volume: - type: OS::Cinder::Volume - properties: - size: { get_param: volume_size } - - agent_volume_att: - type: OS::Cinder::VolumeAttachment + type: OS::Heat::ResourceGroup properties: - instance_uuid: { get_resource: storperf_agent } - volume_id: { get_resource: agent_volume} + count: { get_param: volume_count } + resource_def: { + type: "storperf-volume.yaml", + properties: { + volume_size: { get_param: volume_size }, + agent_instance_uuid: { get_resource: storperf_agent } + } + } outputs: storperf_agent_ip: diff --git a/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml new file mode 100644 index 0000000..aec3393 --- /dev/null +++ b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2018 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +heat_template_version: 2013-05-23 + +parameters: + volume_size: + type: number + description: Size of the volume to be created. + default: 1 + constraints: + - range: { min: 1, max: 1024 } + description: must be between 1 and 1024 Gb. + agent_instance_uuid: + type: string + +resources: + agent_volume: + type: OS::Cinder::Volume + properties: + size: { get_param: volume_size } + + agent_volume_att: + type: OS::Cinder::VolumeAttachment + properties: + instance_uuid: { get_param: agent_instance_uuid } + volume_id: { get_resource: agent_volume} diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py index 7244b66..d54fea3 100644 --- a/docker/storperf-master/storperf/storperf_master.py +++ b/docker/storperf-master/storperf/storperf_master.py @@ -64,10 +64,23 @@ class StorPerfMaster(object): self._agent_flavor = "storperf" self._availability_zone = None self._public_network = None + self._volume_count = 1 self._volume_size = 1 self._cached_stack_id = None self._last_snaps_check_time = None + @property + def volume_count(self): + self._get_stack_info() + return self._volume_count + + @volume_count.setter + def volume_count(self, value): + if (self.stack_id is not None): + raise ParameterError( + "ERROR: Cannot change volume count after stack is created") + self._volume_count = value + @property def volume_size(self): self._get_stack_info() @@ -136,14 +149,12 @@ class StorPerfMaster(object): def _get_stack_info(self): if self._last_snaps_check_time is not None: time_since_check = datetime.now() - self._last_snaps_check_time - if time_since_check.total_seconds() < 30: + if time_since_check.total_seconds() < 60: return self._cached_stack_id self.heat_stack.initialize() if self.heat_stack.get_stack() is not None: self._last_snaps_check_time = datetime.now() - if self._cached_stack_id == self.heat_stack.get_stack().id: - return self._cached_stack_id self._cached_stack_id = self.heat_stack.get_stack().id cinder_cli = cinder_utils.cinder_client(self.os_creds) glance_cli = glance_utils.glance_client(self.os_creds) @@ -162,10 +173,14 @@ class StorPerfMaster(object): image = glance_utils.get_image_by_id(glance_cli, image_id) self._agent_image = image.name - volume_id = server.volume_ids[0]['id'] - volume = cinder_utils.get_volume_by_id( - cinder_cli, volume_id) - self._volume_size = volume.size + self._volume_count = len(server.volume_ids) + if self._volume_count > 0: + volume_id = server.volume_ids[0]['id'] + volume = cinder_utils.get_volume_by_id( + cinder_cli, volume_id) + self.logger.debug("Volume id %s, size=%s" % (volume.id, + volume.size)) + self._volume_size = volume.size router_creators = self.heat_stack.get_router_creators() router1 = router_creators[0] @@ -265,15 +280,17 @@ class StorPerfMaster(object): def create_stack(self): self.stack_settings.resource_files = [ - 'storperf/resources/hot/storperf-agent.yaml'] + 'storperf/resources/hot/storperf-agent.yaml', + 'storperf/resources/hot/storperf-volume.yaml'] self.stack_settings.env_values = self._make_parameters() try: self.heat_stack.create() - except Exception: + except Exception as e: + self.logger.error("Stack creation failed") + self.logger.exception(e) heat_cli = heat_utils.heat_client(self.os_creds) res = heat_utils.get_resources(heat_cli, self.heat_stack.get_stack().id) - self.logger.error("Stack creation failed") reason = "" failed = False for resource in res: @@ -325,10 +342,12 @@ class StorPerfMaster(object): thread.join() self._test_executor.slaves = slaves + self._test_executor.volume_count = self.volume_count params = metadata params['agent_count'] = self.agent_count params['public_network'] = self.public_network + params['volume_count'] = self.volume_count params['volume_size'] = self.volume_size if self.username and self.password: params['username'] = self.username @@ -446,6 +465,7 @@ class StorPerfMaster(object): heat_parameters = {} heat_parameters['public_network'] = self.public_network heat_parameters['agent_count'] = self.agent_count + heat_parameters['volume_count'] = self.volume_count heat_parameters['volume_size'] = self.volume_size heat_parameters['agent_image'] = self.agent_image heat_parameters['agent_flavor'] = self.agent_flavor diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py index 9ed6386..2ed6a9e 100644 --- a/docker/storperf-master/storperf/test_executor.py +++ b/docker/storperf-master/storperf/test_executor.py @@ -53,6 +53,7 @@ class TestExecutor(object): self.job_db = JobDB() self._slaves = [] self._terminated = False + self._volume_count = 1 self._workload_executors = [] self._workload_thread = None self._thread_gate = None @@ -87,6 +88,15 @@ class TestExecutor(object): self.logger.debug("Set slaves to: " + str(slaves)) self._slaves = slaves + @property + def volume_count(self): + return self._volume_count + + @volume_count.setter + def volume_count(self, volume_count): + self.logger.debug("Set volume count to: " + str(volume_count)) + self._volume_count = volume_count + @property def queue_depths(self): return ','.join(self._queue_depths) @@ -282,17 +292,24 @@ class TestExecutor(object): slave_threads = [] for slave in self.slaves: - slave_workload = copy.copy(current_workload['workload']) - slave_workload.remote_host = slave - - self._workload_executors.append(slave_workload) - - t = Thread(target=self.execute_on_node, - args=(slave_workload,), - name="%s worker" % slave) - t.daemon = False - t.start() - slave_threads.append(t) + volume_number = 0 + while volume_number < self.volume_count: + slave_workload = copy.copy(current_workload['workload']) + slave_workload.remote_host = slave + last_char_of_filename = chr(ord( + slave_workload.filename[-1:]) + volume_number) + slave_workload.filename = "%s%s" % \ + (slave_workload.filename[:-1], last_char_of_filename) + self.logger.debug("Device to profile: %s" % + slave_workload.filename) + self._workload_executors.append(slave_workload) + t = Thread(target=self.execute_on_node, + args=(slave_workload,), + name="%s worker" % slave) + t.daemon = False + t.start() + slave_threads.append(t) + volume_number += 1 for slave_thread in slave_threads: self.logger.debug("Waiting on %s" % slave_thread) @@ -335,9 +352,6 @@ class TestExecutor(object): workload.filename = self.filename workload.id = self.job_db.job_id - if (self.filename is not None): - workload.filename = self.filename - if (workload_name.startswith("_")): iodepths = [8, ] blocksizes = [16384, ] diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py index d5282d7..c2c7b7b 100644 --- a/docker/storperf-master/storperf/workloads/_base_workload.py +++ b/docker/storperf-master/storperf/workloads/_base_workload.py @@ -74,9 +74,11 @@ class _base_workload(object): @property def fullname(self): + host_file = self.remote_host+"."+self.filename + host_file = host_file.replace(".", "-").replace("/", "-") return ("%s.%s.queue-depth.%s.block-size.%s.%s" % (str(self.id), self.__class__.__name__, str(self.options['iodepth']), str(self.options['bs']), - str(self.remote_host).replace(".", "-"))) + host_file)) diff --git a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py index e9e47f3..c61fe74 100644 --- a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py +++ b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py @@ -22,33 +22,35 @@ class WorkloadSubclassTest(unittest.TestCase): def test_local_name(self): workload = rr() self.assertEqual(workload.fullname, - "None.rr.queue-depth.1.block-size.64k.None", + "None.rr.queue-depth.1.block-size.64k.None--dev-vdb", workload.fullname) def test_remote_name(self): workload = rw() workload.remote_host = "192.168.0.1" - self.assertEqual(workload.fullname, - "None.rw.queue-depth.1.block-size.64k.192-168-0-1", - workload.fullname) + self.assertEqual( + workload.fullname, + "None.rw.queue-depth.1.block-size.64k.192-168-0-1--dev-vdb", + workload.fullname) def test_blocksize(self): workload = rs() workload.options["bs"] = "4k" self.assertEqual(workload.fullname, - "None.rs.queue-depth.1.block-size.4k.None", + "None.rs.queue-depth.1.block-size.4k.None--dev-vdb", workload.fullname) def test_queue_depth(self): workload = wr() workload.options["iodepth"] = "8" self.assertEqual(workload.fullname, - "None.wr.queue-depth.8.block-size.64k.None", + "None.wr.queue-depth.8.block-size.64k.None--dev-vdb", workload.fullname) def test_id(self): workload = ws() workload.id = "workloadid" - self.assertEqual(workload.fullname, - "workloadid.ws.queue-depth.1.block-size.64k.None", - workload.fullname) + self.assertEqual( + workload.fullname, + "workloadid.ws.queue-depth.1.block-size.64k.None--dev-vdb", + workload.fullname) diff --git a/docs/dev/gerrit.rst b/docs/dev/gerrit.rst deleted file mode 100644 index b227d21..0000000 --- a/docs/dev/gerrit.rst +++ /dev/null @@ -1,203 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) OPNFV, Dell EMC and others. - -====== -Gerrit -====== - -Installing and configuring Git and Git-Review is necessary in order to follow -this guide. The `Getting Started `_ page will provide you with some help for that. - -Committing the code with Gerrit -=============================== - -* Open a terminal window and set the project's directory to the working - directory using the cd command. In this case "/home/tim/OPNFV/storperf" is - the path to the StorPerf project folder on my computer. Replace this with the - path of your own project. - -.. code-block:: bash - - cd /home/tim/OPNFV/storperf - -* Start a new topic for your change. - -.. code-block:: bash - - git checkout -b TOPIC-BRANCH - -* Tell Git which files you would like to take into account for the next commit. - This is called 'staging' the files, by placing them into the staging area, - using the 'git add' command (or the synonym 'git stage' command). - -.. code-block:: bash - - git add storperf/utilities/math.py - git add storperf/tests/utilities/math.py - ... - -* Alternatively, you can choose to stage all files that have been modified - (that is the files you have worked on) since the last time you generated a - commit, by using the -a argument. - -.. code-block:: bash - - git add -a - -* Git won't let you push (upload) any code to Gerrit if you haven't pulled the - latest changes first. So the next step is to pull (download) the latest - changes made to the project by other collaborators using the 'pull' command. - -.. code-block:: bash - - git pull - -* Now that you have the latest version of the project and you have staged the - files you wish to push, it is time to actually commit your work to your local - Git repository. - -.. code-block:: bash - - git commit --signoff -m "Title of change - - Test of change that describes in high level what - was done. There is a lot of documentation in code - so you do not need to repeat it here. - - JIRA: STORPERF-54" - -The message that is required for the commit should follow a specific set of -rules. This practice allows to standardize the description messages attached to -the commits, and eventually navigate among the latter more easily. This -`document `_ happened to be very clear -and useful to get started with that. - - -Pushing the code to Git for review -================================== - -* Now that the code has been comitted into your local Git repository the - following step is to push it online to Gerrit for it to be reviewed. The - command we will use is 'git review'. - -.. code-block:: bash - - git review - -* This will automatically push your local commit into Gerrit, and the command - should get back to you with a Gerrit URL that looks like this : - -.. image:: ../images/git_review.png - -* The OPNFV-Gerrit-Bot in #opnfv-storperf IRC channel will send a message with - the URL as well. - -.. image:: ../images/gerrit_bot.png - -* Copy/Paste the URL into a web browser to get to the Gerrit code review you - have just generated, and click the 'add' button to add reviewers to review - your changes : - -.. image:: ../images/add_reviewers.png - -.. note:: - - Check out this `section `_ if the git - review command returns to you with an "access denied" error. - - -Fetching a Git review -===================== - -If you want to collaborate with another developer, you can fetch their review -by the Gerrit change id (which is part of the URL, and listed in the top left -as Change NNNNN). - -.. code-block:: bash - - git review -d 16213 - -would download the patchset for change 16213. If there were a topic branch -associated with it, it would switch you to that branch, allowing you to look at -different patch sets locally at the same time without conflicts. - - -Modifying the code under review in Gerrit -========================================= - -At the same time the code is being reviewed in Gerrit, you may need to edit it -to make some changes and then send it back for review. The following steps go -through the procedure. - -* Once you have modified/edited your code files under your IDE, you will have - to stage them. The 'status' command is very helpful at this point as it - provides an overview of Git's current state. - -.. code-block:: bash - - git status - -.. image:: ../images/git_status.png - -* The output of the command provides us with the files that have been modified - after the latest commit (in this case I modified storperf/tests/utilities/ - math.py and storperf/utilities/math.py). - -* We can now stage the files that have been modified as part of the Gerrit code - review edition/modification/improvement : - -.. code-block:: bash - - git add storperf/tests/utilities/math.py - git add storperf/utilities/math.py - -* The 'git status' command should take this into consideration : - -.. image:: ../images/git_status_2.png - -* It is now time to commit the newly modified files, but the objective here is - not to create a new commit, we simply want to inject the new changes into the - previous commit. We can achieve that with the '--amend' option on the - 'commit' command : - -.. code-block:: bash - - git commit --amend - -.. image:: ../images/amend_commit.png - -* If the commit was successful, the 'status' command should not return the - updated files as about to be committed. - -* The final step consists in pushing the newly modified commit to Gerrit. - -.. code-block:: bash - - git review - -.. image:: ../images/git_review_2.png - -The Gerrit code review should be updated, which results in a 'patch set 2' -notification appearing in the history log. 'patch set 1' being the original -code review proposition. - - -If Gerrit upload is denied -========================== - -The 'git review' command might return to you with an "access denied" error that -looks like this : - -.. image:: ../images/Access_denied.png - -In this case, you need to make sure your Gerrit account has been added as a -member of the StorPerf contributors group : ldap/opnfv-gerrit-storperf- -contributors. You also want to check that have signed the CLA (Contributor -License Agreement), if not you can sign it in the "Agreements" section of your -Gerrit account : - -.. image:: ../images/CLA_agreement.png - diff --git a/docs/dev/ide.rst b/docs/dev/ide.rst deleted file mode 100644 index 3af4b6c..0000000 --- a/docs/dev/ide.rst +++ /dev/null @@ -1,110 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) OPNFV, Dell EMC and others. - -=== -IDE -=== - -While PyCharm as an excellent IDE, some aspects of it require licensing, and -the PyDev Plugin for Eclipse (packaged as LiClipse) is fully open source -(although donations are welcome). Therefore this section focuses on using -LiClipse for StorPerf development. - - -Download -============ - -.. code-block:: bash - - http://www.liclipse.com/download.html - - -Storperf virtualenv Interpretor -================================= - -Setting up interpreter under PyDev (LiClipse): - -* Go to Project -> Properties, PyDev Interpreter: - -.. image:: ../images/PyDev_Interpreter.jpeg - -* Click to configure an interpreter not listed. - -.. image:: ../images/PyDev_Interpreters_List.jpeg - -* Click New, and create a new interpreter called StorPerf that points to your - Virtual Env. - -.. image:: ../images/PyDev_New_Interpreter.jpeg - -* You should get a pop up similar to: - -.. image:: ../images/PyDev_Interpreter_Folders.jpeg - -* And then you can change the Interpreter to StorPerf. - -.. image:: ../images/PyDev_StorPerf_Interpreter.jpeg - - -Code Formatting -=============== - -Pep8 and Flake8 rule. These are part of the Gerrit checks and I'm going to -start enforcing style guidelines soon. - -* Go to Window -> Preferences, under PyDev, Editor, Code Style, Code Formatter - and select autopep8.py for code formatting. - -.. image:: ../images/Code_formatter.jpeg - -* Next, under Save Actions, enable "Auto-format editor contents before saving", - and "Sort imports on save". - -.. image:: ../images/Save_Actions.jpeg - -* And under Imports, select Delete unused imports. - -.. image:: ../images/Unused_imports.jpeg - -* Go to PyDev -> Editor -> Code Analysis and under the pycodestye.py (pep8), - select Pep8 as Error. This flag highlight badly formatted lines as errors. - These must be fixed before Jenkins will +1 any review. - -.. image:: ../images/Code_analysis.jpeg - - -Import Storperf as Git Project -============================== - -I prefer to do the git clone from the command line, and then import that as a -local project in LiClipse. - -* From the menu: File -> Import Project - -.. image:: ../images/Import_Project.png - -| - -.. image:: ../images/Local_Repo.png - -| - -.. image:: ../images/Add_git.png - -| - -* Browse to the directory where you cloned StorPerf - -.. image:: ../images/Browse.png - -| - -* You should now have storperf as a valid local git repo: - -.. image:: ../images/Git_Selection.png - -| - -* Choose Import as general project - diff --git a/docs/dev/index.rst b/docs/dev/index.rst deleted file mode 100644 index 48000cf..0000000 --- a/docs/dev/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _storperf-devguide: - -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) OPNFV, Dell EMC and others. - -====================== -StorPerf Dev Guide -====================== - -.. toctree:: - :maxdepth: 2 - - initial.rst - ide.rst - unit_tests.rst - gerrit.rst diff --git a/docs/dev/initial.rst b/docs/dev/initial.rst deleted file mode 100644 index 04b1c45..0000000 --- a/docs/dev/initial.rst +++ /dev/null @@ -1,42 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) OPNFV, Dell EMC and others. - -================ -Initial Set up -================ - -Getting the Code -================ - -Replace your LFID with your actual Linux Foundation ID. - -.. code-block:: bash - - git clone ssh://YourLFID@gerrit.opnfv.org:29418/storperf - - -Virtual Environment -======================= -It is preferred to use virtualenv for Python dependencies. This way it is known -exactly what libraries are needed, and can restart from a clean state at any -time to ensure any library is not missing. Simply running the script: - -.. code-block:: bash - - ci/verify.sh - -from inside the storperf directory will automatically create a virtualenv in -the home directory called 'storperf_venv'. This will be used as the Python -interpreter for the IDE. - - -Docker Version -======================= -In order to run the full set of StorPerf services, docker and docker-compose -are required to be installed. This requires docker 17.05 at a minimum. - -.. code-block:: bash - - https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ - diff --git a/docs/dev/unit_tests.rst b/docs/dev/unit_tests.rst deleted file mode 100644 index 98ed3ce..0000000 --- a/docs/dev/unit_tests.rst +++ /dev/null @@ -1,88 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) OPNFV, Dell EMC and others. - -========== -Unit Tests -========== - -Running from CLI -================ - -You technically already did when you ran: - -.. code-block:: bash - - ci/verify.sh - -The shortcut to running the unit tests again from the command line is: - -.. code-block:: bash - - source ~/storperf_venv/bin/activate - nosetests --with-xunit \ - --with-coverage \ - --cover-package=storperf\ - --cover-xml \ - storperf - -.. note:: - - You must be in the top level storperf directory in order to run the tests. - - -Set up under LiClipse -===================== - -Running the tests: - -Right click on the tests folder and select Run as Python Unit Test. Chances -are, you'll get: - -.. code-block:: bash - - Traceback (most recent call last): - File "/home/mark/Documents/EMC/git/opnfv/storperf/storperf/tests/storperf_master_test.py", line 24, in setUp - self.storperf = StorPerfMaster() - File "/home/mark/Documents/EMC/git/opnfv/storperf/storperf/storperf_master.py", line 38, in __init__ - template_file = open("storperf/resources/hot/agent-group.yaml") - IOError: [Errno 2] No such file or directory: 'storperf/resources/hot/agent-group.yaml' - -This means we need to set the working directory of the run configuration. - -* Under the menu: Run -> Run Configurations: - -.. image:: ../images/StorPerf_Tests-Main.jpeg - -* Go to the Arguments tab and change the radio button for Working Directory to - "Default" - -.. image:: ../images/StorPerf_Tests-Arguments.jpeg - -* And on interpreter tab, change the interpreter to StorPerf: - -.. image:: ../images/StorPerf_Tests-Interpreter.jpeg - -* Click Apply. From now on, the run should be clean: - -.. image:: ../images/StorPerf_Tests-Console.jpeg - -| - -.. image:: ../images/StorPerf_Tests-PyUnit.jpeg - - -Adding builtins -=============== - -For some reason, sqlite needs to be added as a builtin. - -* Go to Window -> Preferences, PyDev > Interpreters > Python Interpreter and - select the StorPerf interpreter: - -.. image:: ../images/Python_Interpreters.jpeg - -* Go to the Forced Builtins tab, click New and add sqlite3. - -.. image:: ../images/Forced_Builtins.jpeg - diff --git a/docs/release/release-notes/release-notes.rst b/docs/release/release-notes/release-notes.rst index c0ad0b8..d80e421 100644 --- a/docs/release/release-notes/release-notes.rst +++ b/docs/release/release-notes/release-notes.rst @@ -2,7 +2,7 @@ .. http://creativecommons.org/licenses/by/4.0 -This document provides the release notes for Euphrates 1.0 of StorPerf. +This document provides the release notes for Fraser 2.0 of StorPerf. .. contents:: :depth: 3 @@ -20,14 +20,19 @@ Version history | 2018-04-18 | Fraser 1.0 | Mark Beierl | | | | | | | +--------------------+--------------------+--------------------+--------------------+ +| 2018-05-18 | Fraser 2.0 | Mark Beierl | | +| | | | | ++--------------------+--------------------+--------------------+--------------------+ Important notes ---------------- -This is the release where StorPerf is not delivered as a single container but -is delivered as a series of networked containers. StorPerf must be run using -docker-compose. +StorPerf has added the ability to specify the number of Cinder Volumes per +agent VM to test. The name of the device that the volume is attached to +has been appended to the host IP address in the metrics so that it can be +tracked independently. + Summary -------- @@ -36,10 +41,22 @@ StorPerf is a standalone framework that uses OpenStack to measure Cinder volume performance. If desired, it can push results to the OPNFV Test Results DB, or the embedded Graphite web interface can be used to perform ad hoc queries. -This release changes to docker-compose framework and adds the StorPerf -reporting module. It also marks a change from microsecond (:math:`\mu`\s) to -nano-second (ns) precision for all reported latencies. This is denoted by a change -from lat.mean to lat_ns.mean for read and write metrics. +This release allows for changing of stack attributes from the OpenStack CLI. +Using a command such as + +.. code-block:: + heat stack-update StorPerfAgentGroup --existing -P "agent_count=6" + +will change the existing stack to use 6 agents. Note that StorPerf can take +up to 1 minute after the stack update is complete before detecting the new +values. Please use a GET of the configurations API to test for updated +values prior to submitting a new test. + +The following command changes the number of volumes per agent: + +.. code-block:: + heat stack-update StorPerfAgentGroup --existing -P "volume_count=2" + Release Data ------------- @@ -48,16 +65,16 @@ Release Data | **Project** | StorPerf | | | | +--------------------------------------+--------------------------------------+ -| **Repo/commit-ID** | storperf/fraser.1.0 | +| **Repo/commit-ID** | storperf/fraser.2.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Fraser base release | +| **Release designation** | Fraser base release | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | 2017-10-06 | +| **Release date** | 2018-05-18 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | OPNFV Fraser release 1.0 | +| **Purpose of the delivery** | OPNFV Fraser release 2.0 | | | | +--------------------------------------+--------------------------------------+ @@ -75,7 +92,8 @@ Reason for version Features additions ------------------- -* STORPERF-231 - Integration with SNAPS +* STORPERF-242 - Allow user to change stack parameters outside of StorPerf + Bug Fixes ---------- @@ -87,19 +105,19 @@ Software --------- - `StorPerf master image `_ - (tag: x86_64-fraser.1.0 or aarch64-fraser.1.0) + (tag: x86_64-fraser.2.0 or aarch64-fraser.2.0) - `StorPerf swaggerui `_ - (tag: x86_64-fraser.1.0 or aarch64-fraser.1.0) + (tag: x86_64-fraser.2.0 or aarch64-fraser.2.0) - `StorPerf graphite image `_ - (tag: x86_64-fraser.1.0 or aarch64-fraser.1.0) + (tag: x86_64-fraser.2.0 or aarch64-fraser.2.0) - `StorPerf reporting image `_ - (tag: x86_64-fraser.1.0 or aarch64-fraser.1.0) + (tag: x86_64-fraser.2.0 or aarch64-fraser.2.0) - `StorPerf Http-Frontend image `_ - (tag: x86_64-fraser.1.0 or aarch64-fraser.1.0) + (tag: x86_64-fraser.2.0 or aarch64-fraser.2.0) Documentation -------------- @@ -119,9 +137,17 @@ Known issues * Cirros target VMs do not always mount Cinder volumes on first boot. Sometimes a reboot of the VM is required to properly attach the Cinder volume to /dev/vdb +* A bug in the linux kernel can prevent Cinder volumes from attaching to VMs + using ARM architecture. Specifying the following properties in Glance for + the ARM based image will work around this problem. Note: this will cause + the device to show up as a SCSI device and therefore will be /dev/sdb instead + of /dev/vdb. + +.. code-block: + --property hw_disk_bus=scsi --property hw_scsi_model=virtio-scsi Test Result =========== -- `OPNFV Test Results DB `_ +- `OPNFV Test Results DB `_ diff --git a/docs/testing/developer/devguide/gerrit.rst b/docs/testing/developer/devguide/gerrit.rst new file mode 100644 index 0000000..b227d21 --- /dev/null +++ b/docs/testing/developer/devguide/gerrit.rst @@ -0,0 +1,203 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Dell EMC and others. + +====== +Gerrit +====== + +Installing and configuring Git and Git-Review is necessary in order to follow +this guide. The `Getting Started `_ page will provide you with some help for that. + +Committing the code with Gerrit +=============================== + +* Open a terminal window and set the project's directory to the working + directory using the cd command. In this case "/home/tim/OPNFV/storperf" is + the path to the StorPerf project folder on my computer. Replace this with the + path of your own project. + +.. code-block:: bash + + cd /home/tim/OPNFV/storperf + +* Start a new topic for your change. + +.. code-block:: bash + + git checkout -b TOPIC-BRANCH + +* Tell Git which files you would like to take into account for the next commit. + This is called 'staging' the files, by placing them into the staging area, + using the 'git add' command (or the synonym 'git stage' command). + +.. code-block:: bash + + git add storperf/utilities/math.py + git add storperf/tests/utilities/math.py + ... + +* Alternatively, you can choose to stage all files that have been modified + (that is the files you have worked on) since the last time you generated a + commit, by using the -a argument. + +.. code-block:: bash + + git add -a + +* Git won't let you push (upload) any code to Gerrit if you haven't pulled the + latest changes first. So the next step is to pull (download) the latest + changes made to the project by other collaborators using the 'pull' command. + +.. code-block:: bash + + git pull + +* Now that you have the latest version of the project and you have staged the + files you wish to push, it is time to actually commit your work to your local + Git repository. + +.. code-block:: bash + + git commit --signoff -m "Title of change + + Test of change that describes in high level what + was done. There is a lot of documentation in code + so you do not need to repeat it here. + + JIRA: STORPERF-54" + +The message that is required for the commit should follow a specific set of +rules. This practice allows to standardize the description messages attached to +the commits, and eventually navigate among the latter more easily. This +`document `_ happened to be very clear +and useful to get started with that. + + +Pushing the code to Git for review +================================== + +* Now that the code has been comitted into your local Git repository the + following step is to push it online to Gerrit for it to be reviewed. The + command we will use is 'git review'. + +.. code-block:: bash + + git review + +* This will automatically push your local commit into Gerrit, and the command + should get back to you with a Gerrit URL that looks like this : + +.. image:: ../images/git_review.png + +* The OPNFV-Gerrit-Bot in #opnfv-storperf IRC channel will send a message with + the URL as well. + +.. image:: ../images/gerrit_bot.png + +* Copy/Paste the URL into a web browser to get to the Gerrit code review you + have just generated, and click the 'add' button to add reviewers to review + your changes : + +.. image:: ../images/add_reviewers.png + +.. note:: + + Check out this `section `_ if the git + review command returns to you with an "access denied" error. + + +Fetching a Git review +===================== + +If you want to collaborate with another developer, you can fetch their review +by the Gerrit change id (which is part of the URL, and listed in the top left +as Change NNNNN). + +.. code-block:: bash + + git review -d 16213 + +would download the patchset for change 16213. If there were a topic branch +associated with it, it would switch you to that branch, allowing you to look at +different patch sets locally at the same time without conflicts. + + +Modifying the code under review in Gerrit +========================================= + +At the same time the code is being reviewed in Gerrit, you may need to edit it +to make some changes and then send it back for review. The following steps go +through the procedure. + +* Once you have modified/edited your code files under your IDE, you will have + to stage them. The 'status' command is very helpful at this point as it + provides an overview of Git's current state. + +.. code-block:: bash + + git status + +.. image:: ../images/git_status.png + +* The output of the command provides us with the files that have been modified + after the latest commit (in this case I modified storperf/tests/utilities/ + math.py and storperf/utilities/math.py). + +* We can now stage the files that have been modified as part of the Gerrit code + review edition/modification/improvement : + +.. code-block:: bash + + git add storperf/tests/utilities/math.py + git add storperf/utilities/math.py + +* The 'git status' command should take this into consideration : + +.. image:: ../images/git_status_2.png + +* It is now time to commit the newly modified files, but the objective here is + not to create a new commit, we simply want to inject the new changes into the + previous commit. We can achieve that with the '--amend' option on the + 'commit' command : + +.. code-block:: bash + + git commit --amend + +.. image:: ../images/amend_commit.png + +* If the commit was successful, the 'status' command should not return the + updated files as about to be committed. + +* The final step consists in pushing the newly modified commit to Gerrit. + +.. code-block:: bash + + git review + +.. image:: ../images/git_review_2.png + +The Gerrit code review should be updated, which results in a 'patch set 2' +notification appearing in the history log. 'patch set 1' being the original +code review proposition. + + +If Gerrit upload is denied +========================== + +The 'git review' command might return to you with an "access denied" error that +looks like this : + +.. image:: ../images/Access_denied.png + +In this case, you need to make sure your Gerrit account has been added as a +member of the StorPerf contributors group : ldap/opnfv-gerrit-storperf- +contributors. You also want to check that have signed the CLA (Contributor +License Agreement), if not you can sign it in the "Agreements" section of your +Gerrit account : + +.. image:: ../images/CLA_agreement.png + diff --git a/docs/testing/developer/devguide/ide.rst b/docs/testing/developer/devguide/ide.rst new file mode 100644 index 0000000..3af4b6c --- /dev/null +++ b/docs/testing/developer/devguide/ide.rst @@ -0,0 +1,110 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Dell EMC and others. + +=== +IDE +=== + +While PyCharm as an excellent IDE, some aspects of it require licensing, and +the PyDev Plugin for Eclipse (packaged as LiClipse) is fully open source +(although donations are welcome). Therefore this section focuses on using +LiClipse for StorPerf development. + + +Download +============ + +.. code-block:: bash + + http://www.liclipse.com/download.html + + +Storperf virtualenv Interpretor +================================= + +Setting up interpreter under PyDev (LiClipse): + +* Go to Project -> Properties, PyDev Interpreter: + +.. image:: ../images/PyDev_Interpreter.jpeg + +* Click to configure an interpreter not listed. + +.. image:: ../images/PyDev_Interpreters_List.jpeg + +* Click New, and create a new interpreter called StorPerf that points to your + Virtual Env. + +.. image:: ../images/PyDev_New_Interpreter.jpeg + +* You should get a pop up similar to: + +.. image:: ../images/PyDev_Interpreter_Folders.jpeg + +* And then you can change the Interpreter to StorPerf. + +.. image:: ../images/PyDev_StorPerf_Interpreter.jpeg + + +Code Formatting +=============== + +Pep8 and Flake8 rule. These are part of the Gerrit checks and I'm going to +start enforcing style guidelines soon. + +* Go to Window -> Preferences, under PyDev, Editor, Code Style, Code Formatter + and select autopep8.py for code formatting. + +.. image:: ../images/Code_formatter.jpeg + +* Next, under Save Actions, enable "Auto-format editor contents before saving", + and "Sort imports on save". + +.. image:: ../images/Save_Actions.jpeg + +* And under Imports, select Delete unused imports. + +.. image:: ../images/Unused_imports.jpeg + +* Go to PyDev -> Editor -> Code Analysis and under the pycodestye.py (pep8), + select Pep8 as Error. This flag highlight badly formatted lines as errors. + These must be fixed before Jenkins will +1 any review. + +.. image:: ../images/Code_analysis.jpeg + + +Import Storperf as Git Project +============================== + +I prefer to do the git clone from the command line, and then import that as a +local project in LiClipse. + +* From the menu: File -> Import Project + +.. image:: ../images/Import_Project.png + +| + +.. image:: ../images/Local_Repo.png + +| + +.. image:: ../images/Add_git.png + +| + +* Browse to the directory where you cloned StorPerf + +.. image:: ../images/Browse.png + +| + +* You should now have storperf as a valid local git repo: + +.. image:: ../images/Git_Selection.png + +| + +* Choose Import as general project + diff --git a/docs/testing/developer/devguide/index.rst b/docs/testing/developer/devguide/index.rst new file mode 100644 index 0000000..48000cf --- /dev/null +++ b/docs/testing/developer/devguide/index.rst @@ -0,0 +1,18 @@ +.. _storperf-devguide: + +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Dell EMC and others. + +====================== +StorPerf Dev Guide +====================== + +.. toctree:: + :maxdepth: 2 + + initial.rst + ide.rst + unit_tests.rst + gerrit.rst diff --git a/docs/testing/developer/devguide/initial.rst b/docs/testing/developer/devguide/initial.rst new file mode 100644 index 0000000..04b1c45 --- /dev/null +++ b/docs/testing/developer/devguide/initial.rst @@ -0,0 +1,42 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Dell EMC and others. + +================ +Initial Set up +================ + +Getting the Code +================ + +Replace your LFID with your actual Linux Foundation ID. + +.. code-block:: bash + + git clone ssh://YourLFID@gerrit.opnfv.org:29418/storperf + + +Virtual Environment +======================= +It is preferred to use virtualenv for Python dependencies. This way it is known +exactly what libraries are needed, and can restart from a clean state at any +time to ensure any library is not missing. Simply running the script: + +.. code-block:: bash + + ci/verify.sh + +from inside the storperf directory will automatically create a virtualenv in +the home directory called 'storperf_venv'. This will be used as the Python +interpreter for the IDE. + + +Docker Version +======================= +In order to run the full set of StorPerf services, docker and docker-compose +are required to be installed. This requires docker 17.05 at a minimum. + +.. code-block:: bash + + https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ + diff --git a/docs/testing/developer/devguide/unit_tests.rst b/docs/testing/developer/devguide/unit_tests.rst new file mode 100644 index 0000000..98ed3ce --- /dev/null +++ b/docs/testing/developer/devguide/unit_tests.rst @@ -0,0 +1,88 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Dell EMC and others. + +========== +Unit Tests +========== + +Running from CLI +================ + +You technically already did when you ran: + +.. code-block:: bash + + ci/verify.sh + +The shortcut to running the unit tests again from the command line is: + +.. code-block:: bash + + source ~/storperf_venv/bin/activate + nosetests --with-xunit \ + --with-coverage \ + --cover-package=storperf\ + --cover-xml \ + storperf + +.. note:: + + You must be in the top level storperf directory in order to run the tests. + + +Set up under LiClipse +===================== + +Running the tests: + +Right click on the tests folder and select Run as Python Unit Test. Chances +are, you'll get: + +.. code-block:: bash + + Traceback (most recent call last): + File "/home/mark/Documents/EMC/git/opnfv/storperf/storperf/tests/storperf_master_test.py", line 24, in setUp + self.storperf = StorPerfMaster() + File "/home/mark/Documents/EMC/git/opnfv/storperf/storperf/storperf_master.py", line 38, in __init__ + template_file = open("storperf/resources/hot/agent-group.yaml") + IOError: [Errno 2] No such file or directory: 'storperf/resources/hot/agent-group.yaml' + +This means we need to set the working directory of the run configuration. + +* Under the menu: Run -> Run Configurations: + +.. image:: ../images/StorPerf_Tests-Main.jpeg + +* Go to the Arguments tab and change the radio button for Working Directory to + "Default" + +.. image:: ../images/StorPerf_Tests-Arguments.jpeg + +* And on interpreter tab, change the interpreter to StorPerf: + +.. image:: ../images/StorPerf_Tests-Interpreter.jpeg + +* Click Apply. From now on, the run should be clean: + +.. image:: ../images/StorPerf_Tests-Console.jpeg + +| + +.. image:: ../images/StorPerf_Tests-PyUnit.jpeg + + +Adding builtins +=============== + +For some reason, sqlite needs to be added as a builtin. + +* Go to Window -> Preferences, PyDev > Interpreters > Python Interpreter and + select the StorPerf interpreter: + +.. image:: ../images/Python_Interpreters.jpeg + +* Go to the Forced Builtins tab, click New and add sqlite3. + +.. image:: ../images/Forced_Builtins.jpeg + diff --git a/docs/testing/user/installation.rst b/docs/testing/user/installation.rst index 046da42..b3f0e91 100755 --- a/docs/testing/user/installation.rst +++ b/docs/testing/user/installation.rst @@ -78,8 +78,8 @@ Planning StorPerf is delivered as a series of Docker containers managed by docker-compose. There are two possible methods for installation: -#. Run container on bare metal -#. Run container in a VM +#. Run the containers on bare metal +#. Run the containers in a VM Requirements: @@ -142,7 +142,7 @@ which should result in: .. code-block:: console - 968c0c2d7c0e24f6777c33b37d9b4fd885575155069fb760405ec8214b2eb672 docker-compose.yaml + 69856e9788bec36308a25303ec9154ed68562e126788a47d54641d68ad22c8b9 docker-compose.yaml To run, you must specify two environment variables: @@ -175,7 +175,7 @@ which should result in: .. code-block:: console - 00649e02237d27bf0b40d1a66160a68a56c9f5e1ceb78d7858e30715cf4350e3 create-compose.py + 327cad2a7b3a3ca37910978005c743799313c2b90709e4a3f142286a06e53f57 create-compose.py Note: The script will run fine on python3. Install python future package to avoid error on python2. @@ -187,79 +187,123 @@ Note: The script will run fine on python3. Install python future package to avoi Docker Exec ~~~~~~~~~~~ -If needed, the container can be entered with docker exec. This is not normally +If needed, any StorPerf container can be entered with docker exec. This is not normally required. .. code-block:: console - docker exec -it storperf-master bash + docker exec -it storperf-master /bin/bash -Pulling StorPerf Container -========================== +Pulling StorPerf Containers +=========================== -Master (Euphrates) -~~~~~~~~~~~~~~~~~~ +The tags for StorPerf can be found here: https://hub.docker.com/r/opnfv/storperf-master/tags/ -StorPerf has switched to docker-compose in the latest version. The tag for -pulling the latest master Euphrates container is: +Master (latest) +~~~~~~~~~~~~~~~ -.. code-block:: bash +This tag represents StorPerf at its most current state of development. While +self-tests have been run, there is not a guarantee that all features will be +functional, or there may be bugs. - docker pull opnfv/storperf-master:latest - docker pull opnfv/storperf-reporting:latest - docker pull opnfv/storperf-httpfrontend:latest - docker pull opnfv/storperf-swaggerui:latest - docker pull opnfv/storperf-graphite:latest +Documentation for latest can be found using the latest label at: -However, by itself, this will no longer provide full functionality. Full -instructions are provided in the Running StorPerf Container section of this -document. +http://docs.opnfv.org/en/latest/submodules/storperf/docs/testing/user/index.html +For x86_64 based systems, use: -Danube -~~~~~~ +.. code-block:: console -The tag for the latest stable Danube is be: + TAG=x86_64-latest ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=x86_64-latest ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d -.. code-block:: bash +For 64 bit ARM based systems, use: - docker pull opnfv/storperf:danube.3.0 +.. code-block:: console -Colorado -~~~~~~~~ + TAG=aarch64-latest ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=aarch64-latest ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d -The tag for the latest stable Colorado release is: -.. code-block:: bash +Release (stable) +~~~~~~~~~~~~~~~~ - docker pull opnfv/storperf:colorado.0.1 +This tag represents StorPerf at its most recent stable release. There are +no known bugs and known issues and workarounds are documented in the release +notes. Issues found here should be reported in JIRA: -Brahmaputra -~~~~~~~~~~~ +https://jira.opnfv.org/secure/RapidBoard.jspa?rapidView=3 -The tag for the latest stable Brahmaputra release is: +For x86_64 based systems, use: -.. code-block:: bash +.. code-block:: console - docker pull opnfv/storperf:brahmaputra.1.2 + TAG=x86_64-stable ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=x86_64-stable ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d -StorPerf on ARM Processors -========================== +For 64 bit ARM based systems, use: -StorPerf now supports docker images on ARM processors as well. However, at the moment -there is no upstream image on DockerHub. The user needs to manually build it. Firstly, -clone StorPerf repository from GitHub +.. code-block:: console -.. code-block:: bash + TAG=aarch64-stable ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=aarch64-stable ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d + + + +Fraser (opnfv-6.0.0) +~~~~~~~~~~~~~~~~~~ + +This tag represents the 6th OPNFV release and the 5th StorPerf release. There +are no known bugs and known issues and workarounds are documented in the release +notes. Documentation can be found under the Fraser label at: + +http://docs.opnfv.org/en/stable-fraser/submodules/storperf/docs/testing/user/index.html + +Issues found here should be reported against release 6.0.0 in JIRA: + +https://jira.opnfv.org/secure/RapidBoard.jspa?rapidView=3 + +For x86_64 based systems, use: + +.. code-block:: console + + TAG=x86_64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=x86_64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d + +For 64 bit ARM based systems, use: + +.. code-block:: console + + TAG=aarch64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=aarch64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d + + + +Euphrates (opnfv-5.0.0) +~~~~~~~~~~~~~~~~~ + +This tag represents the 5th OPNFV release and the 4th StorPerf release. There +are no known bugs and known issues and workarounds are documented in the release +notes. Documentation can be found under the Euphrates label at: + +http://docs.opnfv.org/en/stable-euphrates/submodules/storperf/docs/testing/user/index.html + +Issues found here should be reported against release 6.0.0 in JIRA: + +https://jira.opnfv.org/secure/RapidBoard.jspa?rapidView=3 + +For x86_64 based systems, use: + +.. code-block:: console - git clone https://git.opnfv.org/storperf - cd storperf/docker/ + TAG=x86_64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=x86_64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d -Next, build and setup the docker images +For 64 bit ARM based systems, use: .. code-block:: console - TAG=aarch64 ENV_FILE=./admin.rc CARBON_DIR=./carbon docker-compose -f local-docker-compose.yaml build - TAG=aarch64 ENV_FILE=./admin.rc CARBON_DIR=./carbon docker-compose -f local-docker-compose.yaml up -d + TAG=aarch64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=aarch64-opnfv-6.0.0 ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d diff --git a/docs/testing/user/test-usage.rst b/docs/testing/user/test-usage.rst index 8048cff..78bee4e 100644 --- a/docs/testing/user/test-usage.rst +++ b/docs/testing/user/test-usage.rst @@ -36,16 +36,31 @@ Configure The Environment The following pieces of information are required to prepare the environment: -- The number of VMs/Cinder volumes to create -- The Glance image that holds the VM operating system to use. StorPerf has - only been tested with Ubuntu 16.04 -- The OpenStack flavor to use when creating the VMs -- The name of the public network that agents will use -- The size, in gigabytes, of the Cinder volumes to create +- The number of VMs/Cinder volumes to create. +- The Glance image that holds the VM operating system to use. +- The OpenStack flavor to use when creating the VMs. +- The name of the public network that agents will use. +- The size, in gigabytes, of the Cinder volumes to create. +- The number of the Cinder volumes to attach to each VM. - The availability zone (optional) in which the VM is to be launched. Defaults to **nova**. - The username (optional) if we specify a custom image. - The password (optional) for the above image. +**Note**: on ARM based platforms there exists a bug in the kernel which can prevent +VMs from properly attaching Cinder volumes. There are two known workarounds: + +#. Create the environment with 0 Cinder volumes attached, and after the VMs + have finished booting, modify the stack to have 1 or more Cinder volumes. + See section on Changing Stack Parameters later in this guide. +#. Add the following image metadata to Glance. This will cause the Cinder + volume to be mounted as a SCSI device, and therefore your target will be + /dev/sdb, etc, instead of /dev/vdb. You will need to specify this in your + warm up and workload jobs. + +.. code-block: + --property hw_disk_bus=scsi --property hw_scsi_model=virtio-scsi + + The ReST API is a POST to http://StorPerf:5000/api/v1.0/configurations and takes a JSON payload as follows. @@ -57,6 +72,7 @@ takes a JSON payload as follows. "agent_image": string, "public_network": string, "volume_size": int, + "volume_count": int, "availability_zone": string, "username": string, "password": string @@ -165,6 +181,24 @@ is required in order to push results to the OPNFV Test Results DB: "test_case": "snia_steady_state" } +Changing Stack Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~ +While StorPerf currently does not support changing the parameters of the +stack directly, it is possible to change the stack using the OpenStack client +library. The following parameters can be changed: + +- agent_count: to increase or decrease the number of VMs. +- volume_count: to change the number of Cinder volumes per VM. +- volume_size: to increase the size of each volume. Note: Cinder cannot shrink volumes. + +Increasing the number of agents or volumes, or increasing the size of the volumes +will require you to kick off a new _warm_up job to initialize the newly +allocated volumes. + +The following is an example of how to change the stack using the heat client: + +.. code-block:: + heat stack-update StorPerfAgentGroup --existing -P "volume_count=2" Query Jobs Information -- cgit 1.2.3-korg