diff options
57 files changed, 2686 insertions, 768 deletions
diff --git a/cvp/3rd_party/static/testapi-ui/components/home/home.html b/cvp/3rd_party/static/testapi-ui/components/home/home.html index 7b93a246..7dc98566 100644 --- a/cvp/3rd_party/static/testapi-ui/components/home/home.html +++ b/cvp/3rd_party/static/testapi-ui/components/home/home.html @@ -95,10 +95,11 @@ </div> <div class="col-md-10"> <p class="home-content-text"> - The OPNFV Verified program (OVP) demonstrates the readiness and availability of commercial products based on OPNFV. - Verified products and services submitted by vendors and service providers become compliant by implementing explicitly defined interfaces, - behaviors and key features while retaining distinct and value-added innovations across features and capabilities. - The OPNFV Verified program grants products and services with the "OPNFV Verified" mark. Get started with your OVP application today. + The OPNFV Verified program demonstrates the readiness and availability of commercial products based on OPNFV. + Verified products and services submitted by vendors and service providers become compliant by implementing explicitly defined interfaces, + behaviors and key features while retaining distinct and value-added innovations across features and capabilities. + Navigate through the links in the left-hand menu to learn more and get started. You'll find step-step-instructions as well as a participation form. + Use this portal to upload your test results when ready. Please send any questions to verified@opnfv.org<sup><span class="glyphicon glyphicon-envelope" style="font-size:60%;" aria-hidden="true"></span></sup>. </p> </div> </div> diff --git a/cvp/3rd_party/static/testapi-ui/components/results-report/partials/reportDetails.html b/cvp/3rd_party/static/testapi-ui/components/results-report/partials/reportDetails.html index b552ea47..e3632890 100644 --- a/cvp/3rd_party/static/testapi-ui/components/results-report/partials/reportDetails.html +++ b/cvp/3rd_party/static/testapi-ui/components/results-report/partials/reportDetails.html @@ -39,9 +39,9 @@ Test Filters:<br /> <span ng-if="ctrl.testStatus == 'passed'" class="text-success">[{{ value.pass }}]</span> <span ng-if="ctrl.testStatus == 'not passed'" class="text-danger">[{{ value.fail }}]</span> </a> - <a uib-tooltip="view log" ng-click="ctrl.gotoResultLog(area)"><span class="glyphicon glyphicon-cog"></a> + <a uib-tooltip="view log" ng-click="ctrl.gotoResultLog(area)"><span class="glyphicon glyphicon-cog"></span></a> <ul class="list-unstyled" uib-collapse="value.folder"> - <li ng-repeat="case in value.cases"> + <li ng-repeat="case in value.cases" ng-if="(ctrl.testStatus=='passed' && ctrl.case_list.indexOf(case) > -1) || (ctrl.testStatus=='not passed' && ctrl.case_list.indexOf(case) == -1) || ctrl.testStatus=='total'"> <span ng-class="{'glyphicon glyphicon-ok text-success':ctrl.case_list.indexOf(case) > -1, 'glyphicon glyphicon-remove text-warning':ctrl.case_list.indexOf(case) == -1}" aria-hidden="true"></span> <a ng-click="ctrl.gotoDoc(case)">{{ case }}</a> </li> diff --git a/cvp/3rd_party/static/testapi-ui/components/results/results.html b/cvp/3rd_party/static/testapi-ui/components/results/results.html index 8d7e4482..719e6891 100644 --- a/cvp/3rd_party/static/testapi-ui/components/results/results.html +++ b/cvp/3rd_party/static/testapi-ui/components/results/results.html @@ -17,7 +17,7 @@ <div cg-busy="{promise:ctrl.authRequest,message:'Loading'}"></div> <div cg-busy="{promise:ctrl.resultsRequest,message:'Loading'}"></div> -<div ng-show="ctrl.data" class="results-table"> +<div ng-show="ctrl.data" class="results-table" style="width:100%;overflow-x:scroll"> <table ng-data="ctrl.data.result" ng-show="ctrl.data" class="table table-striped table-hover"> <thead> <tr> @@ -29,6 +29,7 @@ <th>Status</th> <th>Log</th> <th>SUT</th> + <th>SUT Version</th> <th class="col-md-2">Operation</th> <th class="col-md-2">Share List</th> </tr> @@ -40,10 +41,11 @@ <td><a uib-tooltip="{{ result.id }}" tooltip-placement="top" tooltip-append-to-body="true" ng-click="ctrl.gotoResultDetail(result.id, result._id)">{{ result.id | limitTo:8 }}</a></td> <td>{{ result.owner }}</td> <td>{{ result.filename || "None"}}</td> - <td><div class="popover-wrapper"><a editable-theme="bs3" onbeforesave="ctrl.changeLabel(result, $data)" editable-text="result.label"> {{ result.label || "None" }}</a></div></td> + <td><div class="popover-wrapper"><a editable-theme="bs3" onbeforesave="ctrl.changeLabel(result, 'label', $data)" editable-text="result.label"> {{ result.label || "None" }}</a></div></td> <td>{{ result.status }}</td> <td><a ng-click="ctrl.downloadLogs(result.id)">logs</a></td> <td><a ng-click="ctrl.gotoSUT(result.id)">info</a></td> + <td><div class="popover-wrapper"><a editable-theme="bs3" onbeforesave="ctrl.changeLabel(result, 'sut_label', $data)" editable-text="result.sut_label"> {{ result.sut_label || "None" }}</a></div></td> <td> <div class="btn-group" uib-dropdown> <a id="single-button" type="button" class="btn btn-success cvp-btn medium accent-color regular-button" uib-dropdown-toggle> diff --git a/cvp/3rd_party/static/testapi-ui/components/results/resultsController.js b/cvp/3rd_party/static/testapi-ui/components/results/resultsController.js index 0b0bbbc8..aa593dc0 100644 --- a/cvp/3rd_party/static/testapi-ui/components/results/resultsController.js +++ b/cvp/3rd_party/static/testapi-ui/components/results/resultsController.js @@ -191,12 +191,12 @@ }); } - function changeLabel(result, data){ - toggleCheck(result, 'label', data); + function changeLabel(result, key, data){ + toggleCheck(result, key, data); } function toReview(result, value){ - var resp = confirm('Once you submit a test result for review, it will become readable to all CVP reviewers. Do you want to proceed?'); + var resp = confirm('Once you submit a test result for review, it will become readable to all OVP reviewers. Do you want to proceed?'); if(resp){ toggleCheck(result, 'status', value); } diff --git a/cvp/opnfv_testapi/resources/test_handlers.py b/cvp/opnfv_testapi/resources/test_handlers.py index 2b5f28c2..77656ae9 100644 --- a/cvp/opnfv_testapi/resources/test_handlers.py +++ b/cvp/opnfv_testapi/resources/test_handlers.py @@ -196,10 +196,16 @@ class TestsGURHandler(GenericTestHandler): return curr_user = self.get_secure_cookie(auth_const.OPENID) - if item in {"shared", "label"}: + if item in {"shared", "label", "sut_label"}: query['owner'] = curr_user db_keys.append('owner') + if item == 'sut_label': + if test['status'] != 'private' and not value: + msg = 'SUT version cannot be changed to None after submitting.' + self.finish_request({'code': 403, 'msg': msg}) + return + if item == "status": if value in {'approved', 'not approved'}: if test['status'] == 'private': @@ -218,6 +224,11 @@ class TestsGURHandler(GenericTestHandler): self.finish_request({'code': 403, 'msg': msg}) return + if not test['sut_label']: + msg = 'Please fill out SUT version before submission' + self.finish_request({'code': 403, 'msg': msg}) + return + query['owner'] = curr_user db_keys.append('owner') diff --git a/cvp/opnfv_testapi/resources/test_models.py b/cvp/opnfv_testapi/resources/test_models.py index 24b4d17b..3829cd6d 100644 --- a/cvp/opnfv_testapi/resources/test_models.py +++ b/cvp/opnfv_testapi/resources/test_models.py @@ -51,9 +51,18 @@ class Test(models.ModelBase): @property trust_indicator: used for long duration test case @ptype trust_indicator: L{TI} """ - def __init__(self, _id=None, owner=None, results=[], - public="false", review="false", status="private", - shared=[], filename="", label="", trust_indicator=None): + def __init__(self, + _id=None, + owner=None, + results=[], + public="false", + review="false", + status="private", + shared=[], + filename="", + label="", + sut_label="", + trust_indicator=None): self._id = _id self.owner = owner self.results = results @@ -64,6 +73,7 @@ class Test(models.ModelBase): self.shared = shared self.filename = filename self.label = label + self.sut_label = sut_label @swagger.model() diff --git a/docs/release/release-notes/dovetail-release.rst b/docs/release/release-notes/dovetail-release.rst new file mode 100644 index 00000000..fa04c60f --- /dev/null +++ b/docs/release/release-notes/dovetail-release.rst @@ -0,0 +1,338 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 + +======= +License +======= + +OPNFV Danube release note for Dovetail Docs +are licensed under a Creative Commons Attribution 4.0 International License. +You should have received a copy of the license along with this. +If not, see <http://creativecommons.org/licenses/by/4.0/>. + +================================================================== +OPNFV Verified Program (OVP) 2018.01 / Dovetail 1.0.0 Release Note +================================================================== + +Abstract +======== + +This document describes the release note of the OPNFV Verified Program and the Dovetail project. + + +Version History +=============== + ++------------+----------+--------------------------+ +| **Date** | **Ver.** | **Comment** | +| | | | ++------------+----------+--------------------------+ +| 2018-01-21 | 1.0.0 | Dovetail for OVP 2018.01 | +| | | Danube release | ++------------+----------+--------------------------+ + +OPNFV Danube Release +==================== + +The OPNFV Verified Program (OVP) allows vendors and operators to obtain 'OPNFV Verified' +status based on an agreed upon set of compliance verification test cases that align to OPNFV +releases. The reference System under Test (SUT) are the NFV components deployed by the OPNFV +installers for a given release, where OVP 2018.01 is based on the Danube release. Participants +of the program can verify commercial or open source offerings against an OVP release. This implies +that the SUT used for verification has interfaces, components, functions and behaviors that align +to OPNFV installer integrations. + +Dovetail is the overall framework used to execute tests and collect results for OVP. Dovetail does +not deliver test content directly. Rather, test content is developed in other OPNFV test frameworks +such as Functest and upstream test communities such as OpenStack's RefStack/Tempest projects. +Dovetail leverages this upstream test content and provides a common set of test platform services +for the OVP. + +Dovetail works in conjunction with a web portal interface dubbed the 'OVP web portal' to allow +users to upload test results to a centralized community repository. This facilitates user +collaboration, result sharing, self-testing and community reviews. It also serves as a hub for +new participants to learn about the program and access key resources. The link for this portal +is at: + + * https://verified.opnfv.org + +Use of the OVP web portal is open to all and only requires a valid Linux Foundation or OpenStack +ID to login. Users are welcome to use the portal to upload, inspect and share results in a private +manner. In order to submit results for official review, the first step is apply for acceptance +into the program with the participation form provided in the link: + + * https://na3.docusign.net/Member/PowerFormSigning.aspx?PowerFormId=579ac00d-0a1f-4db3-82ea-ddd977769a60 + +Test Suites & Test Areas +------------------------ + +OVP/Dovetail groups test cases into test suites and test areas. Test suites are currently a basic +categorization around releases for the most part. Executing the test suite 'ovp.1.0.0' without +further specification will run all the test cases in the OVP 2018.01 release. Test suites are +divided into test areas that can be executed separately. + +Test areas include a division into 'mandatory' and 'optional' in an overarching categorization. +The mandatory test area is further subdivided into the following three test areas, which can +be executed independently: + + * osinterop + * vping + * ha + +The optional test area is further subdivided into the following three test areas: + + * ipv6 + * tempest + * sdnvpn + +All the mandatory test areas are required to be executed with passing results for all inclusive +test cases for results to be reviewed and approved by the community made up of peer reviewers. +The optional test areas are not required to be executed for the official compliance verification +review in the OVP 2018.01 release. However, execution of these areas is encouraged, as some +optional test areas may become mandatory in future releases. + +Test Cases and Sub Test Cases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each test area consists of multiple test cases where each test case can be a single test or +broken down into sub test cases. A listing of test cases with the number of sub test cases noted +in parenthesis is shown below for the OVP 2018.01 release. + +**Mandatory** + * dovetail.osinterop.tc001 (205) + * dovetail.vping.tc001 (1) + * dovetail.vping.tc002 (1) + * dovetail.ha.tc001 (1) + * dovetail.ha.tc002 (1) + * dovetail.ha.tc003 (1) + * dovetail.ha.tc004 (1) + * dovetail.ha.tc005 (1) + * dovetail.ha.tc006 (1) + * dovetail.ha.tc007 (1) + * dovetail.ha.tc008 (1) + +There are a total of 215 mandatory test cases (osinterop: 205, vping: 2, ha: 8). + +**Optional** + * dovetail.ipv6.tc001 (3) + * dovetail.ipv6.tc002 (1) + * dovetail.ipv6.tc003 (1) + * dovetail.ipv6.tc004 (2) + * dovetail.ipv6.tc005 (2) + * dovetail.ipv6.tc006 (1) + * dovetail.ipv6.tc007 (1) + * dovetail.ipv6.tc008 (1) + * dovetail.ipv6.tc009 (1) + * dovetail.ipv6.tc010 (1) + * dovetail.ipv6.tc011 (1) + * dovetail.ipv6.tc012 (1) + * dovetail.ipv6.tc013 (1) + * dovetail.ipv6.tc014 (1) + * dovetail.ipv6.tc015 (1) + * dovetail.ipv6.tc016 (1) + * dovetail.ipv6.tc017 (1) + * dovetail.ipv6.tc018 (1) + * dovetail.ipv6.tc019 (1) + * dovetail.ipv6.tc020 (1) + * dovetail.ipv6.tc021 (1) + * dovetail.ipv6.tc022 (1) + * dovetail.ipv6.tc023 (1) + * dovetail.ipv6.tc024 (1) + * dovetail.ipv6.tc025 (1) + * dovetail.tempest.tc001 (1) + * dovetail.tempest.tc002 (6) + * dovetail.tempest.tc003 (5) + * dovetail.tempest.tc004 (12) + * dovetail.tempest.tc005 (6) + * dovetail.sdnvpn.tc001 (1) + * dovetail.sdnvpn.tc002 (1) + * dovetail.sdnvpn.tc004 (1) + * dovetail.sdnvpn.tc008 (1) + +There are a total of 63 optional test cases (ipv6: 29, tempest: 30, sdnvpn: 4). + +Further details on test area breakdown with expanded test and sub test case names is available at: + + * http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/developer/testscope/index.html + +OPNFV Test Projects and Components +---------------------------------- + +The OPNFV test frameworks integrated into the Dovetail framework that deliver test content are: + + * Functest (leverages OpenStack RefStack/Tempest projects in addition to supplying native test cases) + * Yardstick + +Other upstream components integrated into the Dovetail framework are: + + * TestAPI + * MongoDB + +The above components are part of the OPNFV test collection framework. Further information on how +this framework is used to collect test results can be found at: + + * http://docs.opnfv.org/en/stable-danube/submodules/functest/docs/testing/developer/devguide/#test-collection-framework + +The test frameworks and components above are packaged as Docker containers for Dovetail to employ. +Dovetail creates OVP-specific containers for Functest and TestAPI, while it uses the default +Yardstick Danube container. Additionally, a generic container version of MongoDB is used. +Installation instructions for Dovetail and its dependent containers can be found in the user +guide at: + + * http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/user/userguide/testing_guide.html + +Acceptence and Marketing +------------------------ + +Upon successful community review of results for OVP 2018.01, the OPNFV C&C Committee on behalf of +the Board of Directors can award a product 'OPNFV Verified' status. Use of 'OPNFV Verified' +Program Marks shall be awarded to the platform used for compliance verification. The category label +of 'Infrastructure' is used within the Program Marks logo and limits the scope of this OVP release +to a SUT consisting of NFVI and VIM components using ETSI terminology. It does not provide +compliance verification for specific VNFs in any fashion. The date '2018.01' corresponds to a +reference SUT that aligns to the OPNFV Danube release and currently aligns to the Dovetail +framework version 1.0.0. + +Organizations shall not use the Program Marks in any way that would associate it with any +individual or company logo or brand, beyond the association to the specific platform to which it +was awarded. While OpenStack RefStack interoperability and Tempest integration test cases are +executed as part of the OVP 2018.01 compliance verification test suites, the OVP does not grant or +award OpenStack Marks in any fashion. 'OPNFV Verified' status does not assert readiness for +commercial deployment. + +Please refer to the program governance guidelines and term & conditions documents for additional +details using the respective links: + + * https://www.opnfv.org/wp-content/uploads/sites/12/2018/01/OVP-Governance-Guidelines-1.0.1-012218.pdf + * https://www.opnfv.org/wp-content/uploads/sites/12/2018/01/OVP-Terms-and-Conditions-011918.pdf + +Release Data +============ + ++--------------------------------------+---------------------------------------+ +| **Project** | Dovetail | +| | | ++--------------------------------------+---------------------------------------+ +| **Repo tag** | ovp.1.0.0 | +| | | ++--------------------------------------+---------------------------------------+ +| **Release designation** | OPNFV Verified Program (OVP) | +| | 2018.01 (Danube) | ++--------------------------------------+---------------------------------------+ +| **Release date** | January 21st 2018 | +| | | ++--------------------------------------+---------------------------------------+ +| **Purpose of the delivery** | Support OVP 2018.01 release with | +| | OPNFV Danube release as reference SUT | ++--------------------------------------+---------------------------------------+ + +Deliverables +============ + +Software +-------- + ++-----------------+----------------------+-------------+ +| Docker | Docker Image | Tag | +| Container | | | ++=================+======================+=============+ +| dovetail | opnfv/dovetail | ovp.1.0.0 | ++-----------------+----------------------+-------------+ +| functest | opnfv/functest | ovp.1.0.0 | ++-----------------+----------------------+-------------+ +| yardstick | opnfv/yardstick | danube.3.2 | ++-----------------+----------------------+-------------+ +| testapi | opnfv/testapi | ovp.1.0.0 | ++-----------------+----------------------+-------------+ +| mongo | mongo | 3.2.1 | ++-----------------+----------------------+-------------+ + + + - Dovetail Docker images: https://hub.docker.com/r/opnfv/dovetail + + - Functest Docker images: https://hub.docker.com/r/opnfv/functest + + - Yardstick Docker images: https://hub.docker.com/r/opnfv/yardstick + + - TestAPI Docker images: https://hub.docker.com/r/opnfv/testapi + + - MongoDB Docker images: https://hub.docker.com/r/mongo + + +Documents +--------- + + - System Preparation Guide: http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/user/systempreparation/index.html + + - User Guide: http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/user/userguide/testing_guide.html + + - Test Specifications: http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/user/testspecification/index.html + + - Dovetail CLI Reference: http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/user/userguide/cli_reference.html + + - Process Workflow: http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/user/certificationworkflow/index.html + + - Reviewer Guide: http://docs.opnfv.org/en/stable-danube/submodules/dovetail/docs/testing/user/reviewerguide/index.html + + +Version Change +============== + +This is the first major release of OVP/Dovetail. Please refer to the link below for minor +version changes during pre-release and beta phases. + + * https://wiki.opnfv.org/display/dovetail/Running+history+for+the+dovetail+tool + +Testing with OPNFV Danube Installers +==================================== + +OVP 2018.01 and Dovetail 1.0.0 are known to be have been tested with the following OPNFV +Danube installer versions. + ++-----------------+----------------------+ +| Installer | Version | ++=================+======================+ +| Apex | danube.3.1 | ++-----------------+----------------------+ +| Compass | danube.3.1 | ++-----------------+----------------------+ +| Fuel | danube.3.0 | ++-----------------+----------------------+ + + +Danube Known Restrictions/Issues +================================== + +Please refer to the following link for known issues with the Dovetail Danube release: + + * https://wiki.opnfv.org/display/dovetail/Running+history+for+the+dovetail+tool#Runninghistoryforthedovetailtool-4.KnownIssuesList + +Open JIRA Tickets +================= + ++------------------+-----------------------------------------------+ +| JIRA | Description | ++==================+===============================================+ +| | | +| | | ++------------------+-----------------------------------------------+ + +All blocking tickets have been fixed. + + +Useful Links +============ + + - OVP Web Portal: https://verified.opnfv.org + + - Wiki Project Page: https://wiki.opnfv.org/display/dovetail + + - Dovetail Repo: https://git.opnfv.org/dovetail/ + + - Dovetail CI dashboard: https://build.opnfv.org/ci/view/dovetail/ + + - JIRA dashboard: https://jira.opnfv.org/secure/RapidBoard.jspa?rapidView=149 + + - Dovetail IRC Channel: #opnfv-dovetail + + - Dovetail Test Configuration: https://git.opnfv.org/dovetail/tree/dovetail/compliance/ovp.1.0.0.yml diff --git a/docs/release/release-notes/index.rst b/docs/release/release-notes/index.rst new file mode 100644 index 00000000..ad4cade0 --- /dev/null +++ b/docs/release/release-notes/index.rst @@ -0,0 +1,20 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 + +.. _dovetail-releasenotes: + +********************** +Dovetail Release Notes +********************** + +.. toctree:: + :numbered: + :maxdepth: 4 + + dovetail-release.rst + +Revision: _sha1_ + +:Author: Eddie Arrage (eddie.arrage@huawei.com) + +Build date: |today| diff --git a/docs/testing/developer/testcaserequirements/index.rst b/docs/testing/developer/testcaserequirements/index.rst index daed9181..9084462e 100644 --- a/docs/testing/developer/testcaserequirements/index.rst +++ b/docs/testing/developer/testcaserequirements/index.rst @@ -5,21 +5,21 @@ .. _dovetail-test_case_requirements: ========================================================== -Compliance Verification Program test case requirements +OPNFV Verified Program test case requirements ========================================================== .. toctree:: :maxdepth: 2 -CVP Test Suite Purpose and Goals +OVP Test Suite Purpose and Goals ================================ -The CVP test suite is intended to provide a method for validating the +The OVP test suite is intended to provide a method for validating the interfaces and behaviors of an NFVi platform according to the expected capabilities exposed in OPNFV. The behavioral foundation evaluated in these tests should serve to provide a functional baseline for VNF deployment and -portability across NFVi instances. All CVP tests are available in open source +portability across NFVi instances. All OVP tests are available in open source and are executed in open source test frameworks. @@ -27,7 +27,7 @@ Test case requirements ====================== The following requirements are mandatory for a test to be submitted for -consideration in the CVP test suite: +consideration in the OVP test suite: - All test cases must be fully documented, in a common format. Please consider the existing :ref:`dovetail-test_case_specification` as examples. @@ -35,10 +35,10 @@ consideration in the CVP test suite: - Clearly identifying the test procedure and expected results / metrics to determine a “pass” or “fail” result. -- Tests must be validated for the purpose of CVP, tests should be run with both +- Tests must be validated for the purpose of OVP, tests should be run with both an expected positive and negative outcome. -- At the current stage of CVP, only functional tests are eligible, performance +- At the current stage of OVP, only functional tests are eligible, performance testing is out of scope. - Performance test output could be built in as “for information only”, but @@ -81,7 +81,7 @@ consideration in the CVP test suite: - applicable, i.e., support the feature that the test exercises, and - - released, i.e., in the OPNFV release supported by the CVP test suite + - released, i.e., in the OPNFV release supported by the OVP test suite version. - Tests must run against a fully deployed and operational system under test. @@ -109,9 +109,9 @@ consideration in the CVP test suite: - Fault/Error test case descriptions - Post conditions where the system state may be left changed after completion -New test case proposals should complete a CVP test case worksheet to ensure +New test case proposals should complete a OVP test case worksheet to ensure that all of these considerations are met before the test case is approved for -inclusion in the CVP test suite. +inclusion in the OVP test suite. Dovetail Test Suite Naming Convention diff --git a/docs/testing/developer/testscope/index.rst b/docs/testing/developer/testscope/index.rst index 218e74c8..ed5d2e3d 100644 --- a/docs/testing/developer/testscope/index.rst +++ b/docs/testing/developer/testscope/index.rst @@ -12,7 +12,7 @@ Compliance and Verification program accepted test cases :maxdepth: 2 -Mandatory CVP Test Areas +Mandatory OVP Test Areas ======================== ---------------------------------- @@ -82,7 +82,7 @@ Basic server operations in the Compute API | tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image | tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit | tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name -| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_active_status | tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard | tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date | tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date @@ -120,8 +120,6 @@ Basic server operations in the Compute API | tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details | tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus | tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details -| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status -| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server Retrieve volume information through the Compute API @@ -166,33 +164,22 @@ Image deletion tests using the Glance v2 API Image get tests using the Glance v2 API --------------------------------------- -| tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema -| tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema +| tempest.api.image.v2.test_images.ListUserImagesTest.test_get_image_schema +| tempest.api.image.v2.test_images.ListUserImagesTest.test_get_images_schema | tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image | tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id | tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image -| tempest.api.image.v2.test_images.ListUserImagesTest.test_get_image_schema -| tempest.api.image.v2.test_images.ListUserImagesTest.test_get_images_schema CRUD image operations in Images API v2 -------------------------------------- -| tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params -| tempest.api.image.v2.test_images.ListImagesTest.test_index_no_params | tempest.api.image.v2.test_images.ListUserImagesTest.test_list_no_params Image list tests using the Glance v2 API ---------------------------------------- -| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format -| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format -| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit -| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size -| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size -| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status -| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility | tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_container_format | tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_disk_format | tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_limit @@ -244,24 +231,6 @@ Basic CRUD operations on L2 networks and L2 network ports | tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields | tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes | tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes -| tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_allocation_pools -| tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_dhcp_enabled -| tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_gw -| tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_gw_and_allocation_pools -| tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_host_routes_and_dns_nameservers -| tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_without_gateway -| tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_all_attributes -| tempest.api.network.test_networks.NetworksTestJSON.test_create_update_delete_network_subnet -| tempest.api.network.test_networks.NetworksTestJSON.test_delete_network_with_subnet -| tempest.api.network.test_networks.NetworksTestJSON.test_list_networks -| tempest.api.network.test_networks.NetworksTestJSON.test_list_networks_fields -| tempest.api.network.test_networks.NetworksTestJSON.test_list_subnets -| tempest.api.network.test_networks.NetworksTestJSON.test_list_subnets_fields -| tempest.api.network.test_networks.NetworksTestJSON.test_show_network -| tempest.api.network.test_networks.NetworksTestJSON.test_show_network_fields -| tempest.api.network.test_networks.NetworksTestJSON.test_show_subnet -| tempest.api.network.test_networks.NetworksTestJSON.test_show_subnet_fields -| tempest.api.network.test_networks.NetworksTestJSON.test_update_subnet_gw_dns_host_routes_dhcp Basic CRUD operations on security groups @@ -305,14 +274,12 @@ Volume service availability zone operations with the Cinder v2 API ------------------------------------------------------------------ | tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list -| tempest.api.volume.test_availability_zone.AvailabilityZoneTestJSON.test_get_availability_zone_list Volume cloning operations with the Cinder v2 API ------------------------------------------------ | tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone -| tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_as_clone Image copy-to-volume operations with the Cinder v2 API @@ -320,8 +287,6 @@ Image copy-to-volume operations with the Cinder v2 API | tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable | tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image -| tempest.api.volume.test_volumes_get.VolumesActionsTest.test_volume_bootable -| tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_from_image Volume creation and deletion operations with the Cinder v2 API @@ -331,24 +296,15 @@ Volume creation and deletion operations with the Cinder v2 API | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type -| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_without_passing_size | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero -| tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_invalid_size -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_source_volid -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_volume_type -| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_without_passing_size -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_without_passing_size -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_negative -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_zero Volume service extension listing operations with the Cinder v2 API ------------------------------------------------------------------ | tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions -| tempest.api.volume.test_extensions.ExtensionsTestJSON.test_list_extensions Volume GET operations with the Cinder v2 API @@ -357,9 +313,6 @@ Volume GET operations with the Cinder v2 API | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_invalid_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_volume_without_passing_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_get_nonexistent_volume_id Volume listing operations with the Cinder v2 API @@ -384,43 +337,19 @@ Volume listing operations with the Cinder v2 API | tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination | tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params | tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_by_name -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_by_name -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_param_display_name_and_status -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_display_name_and_status -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_metadata -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_details -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_param_metadata -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_by_availability_zone -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_by_status -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_by_availability_zone -| tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_by_status -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_invalid_status -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_nonexistent_name -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_invalid_status -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_nonexistent_name -| tempest.api.volume.v2.test_volumes_list.VolumesListTestJSON.test_volume_list_details_pagination -| tempest.api.volume.v2.test_volumes_list.VolumesListTestJSON.test_volume_list_details_with_multiple_params -| tempest.api.volume.v2.test_volumes_list.VolumesListTestJSON.test_volume_list_pagination Volume metadata operations with the Cinder v2 API ------------------------------------------------- -| tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata -| tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item -| tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_crud_volume_metadata | tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_crud_volume_metadata -| tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_update_volume_metadata_item -| tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_update_show_volume_metadata_item +| tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item Verification of read-only status on volumes with the Cinder v2 API ------------------------------------------------------------------ | tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update -| tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_readonly_update Volume reservation operations with the Cinder v2 API @@ -430,16 +359,12 @@ Volume reservation operations with the Cinder v2 API | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_negative_volume_status | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_nonexistent_volume_id | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_unreserve_volume_with_nonexistent_volume_id -| tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_reserve_unreserve_volume -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_reserve_volume_with_negative_volume_status -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_reserve_volume_with_nonexistent_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_unreserve_volume_with_nonexistent_volume_id Volume snapshot creation/deletion operations with the Cinder v2 API ------------------------------------------------------------------- -| tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata +| tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_crud_snapshot_metadata | tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id @@ -447,26 +372,10 @@ Volume snapshot creation/deletion operations with the Cinder v2 API | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id | tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete | tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot -| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params -| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params -| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id -| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id -| tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_crud_snapshot_metadata -| tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_crud_snapshot_metadata -| tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_update_snapshot_metadata_item -| tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_update_show_snapshot_metadata_item -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_snapshot_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_invalid_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_volume_without_passing_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_delete_nonexistent_volume_id -| tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_get_list_update_delete -| tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_volume_from_snapshot -| tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_details_with_params | tempest.api.volume.test_volumes_snapshots_list.VolumesV2SnapshotListTestJSON.test_snapshots_list_details_with_params -| tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_with_params | tempest.api.volume.test_volumes_snapshots_list.VolumesV2SnapshotListTestJSON.test_snapshots_list_with_params -| tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id -| tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id +| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id +| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id Volume update operations with the Cinder v2 API @@ -475,9 +384,6 @@ Volume update operations with the Cinder v2 API | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id | tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_empty_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_invalid_volume_id -| tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_nonexistent_volume_id --------------------------- @@ -485,27 +391,27 @@ Test Area High Availability --------------------------- Verify high availability of OpenStack controller services ------------------------------------------------------- +--------------------------------------------------------- -| opnfv.ha.tc001.nova-api_service_down -| opnfv.ha.tc003.neutron-server_service_down -| opnfv.ha.tc004.keystone_service_down -| opnfv.ha.tc005.glance-api_service_down -| opnfv.ha.tc006.cinder-api_service_down -| opnfv.ha.tc009.cpu_overload -| opnfv.ha.tc010.disk_I/O_block -| opnfv.ha.tc011.load_balance_service_down +| dovetail.ha.tc001.nova-api_service_down +| dovetail.ha.tc002.neutron-server_service_down +| dovetail.ha.tc003.keystone_service_down +| dovetail.ha.tc004.glance-api_service_down +| dovetail.ha.tc005.cinder-api_service_down +| dovetail.ha.tc006.cpu_overload +| dovetail.ha.tc007.disk_I/O_overload +| dovetail.ha.tc008.load_balance_service_down ---------------------------------------- Test Area vPing - Basic VNF Connectivity ---------------------------------------- -| opnfv.vping.userdata -| opnfv.vping.ssh +| dovetail.vping.tc001.userdata +| dovetail.vping.tc002.ssh -Optional CVP Test Areas +Optional OVP Test Areas ======================== @@ -516,10 +422,10 @@ Test Area BGP VPN Verify association and dissasocitation of node using route targets ------------------------------------------------------------------ -| opnfv.sdnvpn.subnet_connectivity -| opnfv.sdnvpn.tenant separation -| opnfv.sdnvpn.router_association -| opnfv.sdnvpn.router_association_floating_ip +| dovetail.sdnvpn.tc001.subnet_connectivity +| dovetail.sdnvpn.tc002.tenant_separation +| dovetail.sdnvpn.tc004.router_association +| dovetail.sdnvpn.tc008.router_association_floating_ip -------------------------------------------------- IPv6 Compliance Testing Methodology and Test Cases @@ -681,7 +587,6 @@ Correct Behavior after Common Virtual Machine Life Cycles Events | tempest.scenario.test_minimum_basic.TestMinimumBasicScenario.test_minimum_basic_scenario | tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration -| tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert | tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_pause_unpause | tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_reboot | tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_rebuild diff --git a/docs/testing/user/certificationworkflow/ApplicationForm.rst b/docs/testing/user/certificationworkflow/ApplicationForm.rst index c3d3e7eb..aac9a46e 100644 --- a/docs/testing/user/certificationworkflow/ApplicationForm.rst +++ b/docs/testing/user/certificationworkflow/ApplicationForm.rst @@ -2,11 +2,9 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, Intel Corporation and others. -.._cvp-application-form: - -====================================================== -OPNFV COMPLIANCE VERIFICATION PROGRAM APPLICATION FORM -====================================================== +======================================= +OPNFV Verified Program Application Form +======================================= +----------------------------------+--------------------------------------------------------------------------------------------+ @@ -48,7 +46,7 @@ OPNFV COMPLIANCE VERIFICATION PROGRAM APPLICATION FORM | +--------------------------------------------------------------------------------------------+ | | | +----------------------------------+--------------------------------------------------------------------------------------------+ -| Primary business email | *Only the Business email address should be used for official communication with OPNFV CVP* | +| Primary business email | *Only the Business email address should be used for official communication with OPNFV OVP* | | +--------------------------------------------------------------------------------------------+ | | | | +--------------------------------------------------------------------------------------------+ @@ -62,7 +60,7 @@ OPNFV COMPLIANCE VERIFICATION PROGRAM APPLICATION FORM | +--------------------------------------------------------------------------------------------+ | | | +----------------------------------+--------------------------------------------------------------------------------------------+ -| User ID for CVP web portal | *Choose one: (i) Linux Foundation (ii) Openstack (iii) Github (iv) Google (v) Fackbook ID* | +| User ID for OVP web portal | *Choose one: (i) Linux Foundation (ii) Openstack (iii) Github (iv) Google (v) Facebook ID* | | +--------------------------------------------------------------------------------------------+ | | | | +--------------------------------------------------------------------------------------------+ diff --git a/docs/testing/user/certificationworkflow/index.rst b/docs/testing/user/certificationworkflow/index.rst index 022b2c2e..59cd8982 100644 --- a/docs/testing/user/certificationworkflow/index.rst +++ b/docs/testing/user/certificationworkflow/index.rst @@ -4,28 +4,28 @@ .. _dovetail-certification_workflow: -================================================================ -OPNFV Compliance Verification Program certification workflow -================================================================ +=============================== +OPNFV Verified Program Workflow +=============================== Introduction ============ -This document provides guidance for testers on how to obtain OPNFV compliance -certification. The OPNFV Compliance Verification Program (CVP) is administered by -the OPNFV Compliance and Certification (C&C) committee. +This document provides guidance for prospective participants on how to obtain 'OPNFV Verified' +status. The OPNFV Verified Program (OVP) is administered by the OPNFV Compliance and Certification +(C&C) committee. For further information about the workflow and general inquiries about the -program, please check out the `CVP web portal`_, or contact -the C&C committee by email address cvp@opnfv.org. This email address should be used -for all communication with the CVP. +program, please check out the `OVP web portal`_, or contact +the C&C committee by email address verified@opnfv.org. This email address should be used +for all communication with the OVP. -Step 1: Applying -================ +Step 1: Participation Form Submission +===================================== -A tester should start the process by completing an application. -The application form can found on the `CVP web portal`_ and the following -information should be provided: +A participant should start the process by submitting an online participation form. The participation +form can found on the `OVP web portal`_ or directly at `OVP participation form`_ and the +following information must be provided: - Organization name - Organization website (if public) @@ -35,26 +35,25 @@ information should be provided: - Product categories, choose one: (i) software and hardware (ii) software and third party hardware (please specify) - Primary contact name, business email, postal address and phone number - Only the primary email address should be used for - official communication with OPNFV CVP. -- User ID for CVP web portal - The CVP web portal supports the Linux Foundation user ID in the current release. + Only the primary contact email address should be used for + official communication with OPNFV OVP. +- User ID for OVP web portal + The OVP web portal supports the Linux Foundation user ID in the current release. If a new user ID is needed, visit https://identity.linuxfoundation.org. - Location where the verification testing is to be conducted. Choose one: (internal vendor lab, third-party lab) - If the test is to be conducted by a third-party lab, please specify name and contact information of the third-party lab, including email, address and phone number. +- OVP software version for compliance verification +- Testing date -Please email the completed application using the primary contact email -account in order to establish identity. - -Once the application information is received and in order, an email response will be -sent to the primary contact with confirmation and information to proceed. +Once the participation form information is received and in order, an email response will be +sent to the primary contact with confirmation and information to proceed. The primary contact +specified in the participation form will be entered into OVP web portal back-end by the program +administrator and will be permitted to submit results for review on behalf of their organization. -[Editor's note: -No fee has been established at this time for CVP applications. Recommend -we skip fee for the initial release of CVP.] +There is no fee at this time for participation in the OVP. Step 2: Testing =============== @@ -65,56 +64,61 @@ The following documents guide testers to prepare the test environment and run te - :ref:`dovetail-test_case_specification` - :ref:`dovetail-testing_user_guide` -A unique Test ID is generated by the Dovetail tool for each test run. Please take a note -of this ID for future reference. +A unique Test ID is generated by the Dovetail tool for each test run and can only be +submitted to the OVP web portal once. Step 3: Submitting Test Results =============================== -Testers can upload the test results to the `CVP web portal`_. -By default, the results are visible only to the tester who uploaded the data. +Users/testers other than the primary contact may use the OVP web portal as a resource to upload, +evaluate and share results in a private manner. Testers can upload the test results to the +`OVP web portal`_. By default, the results are visible only to the tester who uploaded the data. Testers can self-review the test results through the portal until they are ready to ask -for CVP review. They may also update with or add new test results as needed. +for OVP review. They may also add new test results as needed. -Once the tester is satisfied with the test result, the tester grants access to the test result -for CVP review via the portal. The test result is identified by the unique Test ID. +Once the tester is satisfied with the test result, the primary contact grants access to the test +result for OVP review using a 'submit for review' operation via the portal. The test result is +identified by the unique Test ID and becomes visible to a review group comprised of OPNFV +community members. -When a test result is made visible to the reviewers, the web portal will notify -cvp@opnfv.org and Cc the primary contact email that a review request has been made and reference -the Test ID. This will alert the C&C Committee to start the CVP review process. +When a test result is made visible to the reviewers, the program administrator will ask for +volunteers from the review group using the verified@opnfv.org email and CC the primary contact +email that a review request has been made. The program administrator will supply the Test ID +and owner field (primary contact user ID) to the reviewers to identify the results. -Step 4: CVP Review +Step 4: OVP Review =================== -Upon receiving the email notification and the Test ID, the C&C Committee conducts a -peer based review of the test result. Persons employed by the same organization that submitted -the test results or by affiliated organizations will not be part of the reviewers. +Upon receiving the email request from the program administrator, the review group conducts a +peer based review of the test result using reviewer guidelines published per OVP release. +Persons employed by the same organization that submitted the test results or by affiliated +organizations will not be part of the reviewers. -The primary contact may be asked via email for any missing information or -clarification of the application. The reviewers will make a determination and recommend -compliance or non-compliance to -the C&C Committee. Normally, the outcome of the review should be communicated -to the tester within 10 business days after all required information is in order. +The primary contact may be asked via email for any missing information or clarification of the +test results. The reviewers will make a determination and recommend compliance or non-compliance +to the C&C Committee. A positive review requires a minimum of two approvals from two distinct +organizations without any negative reviews. The program administrator sends an email to OVP/C&C +emails announcing a positive review. A one week limit is given for issues to be raised. If no +issue is raised, the C&C Committee approves the result and the program administrator sends an +email to OVP/C&C emails stating the result is approved. -If an application is denied, an appeal can be made to the C&C Committee or ultimately to the -Board of Directors of OPNFV. +Normally, the outcome of the review should be communicated to the primary contact within 10 +business days after all required information is in order. -Step 5: Grant of Use of Logo -============================ +If a test result is denied, an appeal can be made to the C&C Committee for arbitration. -If an application is approved, further information will be communicated to the tester -on the guidelines of using OPNFV CVP logos and the status of compliance for promotional purposes. +Step 5: Grant of Use of Program Marks +===================================== - -Appendix -======== +If an application is approved, further information will be communicated to the primary contact +on the guidelines of using OVP Program Marks (including OVP logo) and the status of compliance +for promotional purposes. .. toctree:: :maxdepth: 2 - ApplicationForm - .. References -.. _`CVP web portal`: https://cvp.opnfv.org +.. _`OVP web portal`: https://verified.opnfv.org +.. _`OVP participation form`: https://na3.docusign.net/Member/PowerFormSigning.aspx?PowerFormId=579ac00d-0a1f-4db3-82ea-ddd977769a60 diff --git a/docs/testing/user/index.rst b/docs/testing/user/index.rst deleted file mode 100644 index 120aba45..00000000 --- a/docs/testing/user/index.rst +++ /dev/null @@ -1,71 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) Ericsson AB - -====================================================== -Compliance and Verification program user documentation -====================================================== - -.. toctree:: - :maxdepth: 2 - -Version history -=============== - -+------------+----------+------------------+----------------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+----------------------------------+ -| 2017-03-15 | 0.0.1 | Chris Price | Draft version | -| | | | | -+------------+----------+------------------+----------------------------------+ - - -Introduction -============ - -This document provides all relevant user documentation for executing the OPNFV -compliance and certification program (CVP). The CVP provides mechanisms for evaluating -NFV infrastructures to establish their alignment with the target architectures, behaviours -and functions defined by the OPNFV community. - -Compliance and Verification Program guidelines -============================================== - -The CVP provides publicly available mechanisms for evaluating compliance to OPNFV concepts, -behaviours and architectures. The CVP, through the Compliance and Certification -committee, is responsible for evaluating the results of any evaluations and providing -access to any associated OPNFV branding. Details of the CVP are described in the `CVP Document.`_ - -"Add the final CVP doc URL in plain text here" - -Compliance and Verification program test specification -====================================================== - -The `test specification`_ provides an outline of the prerequisites, methods, tools -& areas that are applicable to the CVP test process. The `test specification`_ is intended -to give a use a comprehensive overview of what is expected of engineer prior to running -the NFV platform evaluation suite, what tools and processes will be used during the -evaluation and a guide as to how the results of the tests will be interpreted. It is advised -that all practitioners of the CVP read the `test specification`_ prior to beginning -the process of evaluation. - -"Add the final test spec URL in plain text here" - -Compliance and Verification program Test Plan -============================================= - -The test spec will outline the processes & areas and the test case descriptions included. -This document will in addition outline how to modularise the areas to be executed, how to -run each area and when the testing process begins and ends. - -Compliance and Verification program Test results -================================================ - -To be added once our test result reporting system and processes are defined. - -References -========== - -.. _`CVP Document.`: https://opnfv.org -.. _`test specification`: https://opnfv.org diff --git a/docs/testing/user/ovpaddendum/exemption-strict-API-validation.rst b/docs/testing/user/ovpaddendum/exemption-strict-API-validation.rst new file mode 100644 index 00000000..58b9283f --- /dev/null +++ b/docs/testing/user/ovpaddendum/exemption-strict-API-validation.rst @@ -0,0 +1,185 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) Ericsson and others + +.. _dovetail-exemption_process_api_response_validation: + +========================================== +Disabling Strict API Validation in Tempest +========================================== + +.. toctree:: + :maxdepth: 2 + + +Introduction +============ + +In 2015, the OpenStack QA team introduced a validation mechanism for Nova API +responses in Tempest [1]_ with the goal of enforcing Nova API micro-versions. +The API validation mechanism verifies that API responses only contain data +elements (properties) as explicitly defined in API response schemas [2]_. In +case additional data elements are found in Nova API responses, the +corresponding tests fail immediately without asserting whether or not the +particular API operation actually succeeded or not. + +Independently, cloud vendors have extended their commercial OpenStack cloud +implementations with additional functionality which requires API extensions. +Consequently, such cloud implementations do not pass Tempest tests which +validate API responses despite actually implementing and providing the tested +functionality. + +This document describes an exemption process for use within the OPNFV Verified +Program which + +i) allows vendors to pass Tempest tests if the tested functionality is + fully supported despite the presence of additional data elements in API + responses, and + +ii) makes the application of the exemption process transparently visible in + test results. + + +Background and benefits for OVP +=============================== + +Vendors of commercial NFV products have extended OpenStack to provide +additional (NFV) functionality to their customers and to fill functional gaps +in OpenStack. These add-ons potentially extend the OpenStack API in two ways: + +i) new API endpoints and + +ii) additional attributes returned by existing API endpoints. + +New API endpoints typically go unnoticed by OpenStack Tempest tests and hence +do not interfere with existing tests. In contrast, (Nova) Tempest tests +actively validate the responses returned by existing API endpoints against +pre-defined schemas. An API response is considered invalid if additional +attributes are present (see example below). Hence, this particular type of +functional extension of OpenStack causes existing Tempest tests to fail, +irrespective of whether or not the functionality which is supposed to be tested +is actually available. As a result, a Tempest test failing due to extended API +responses does not provide information about whether the tested functionality +is available or not. + +The OPNFV Verified Program has inherited the policy to strictly validate API +responses from OpenStack by including a selection of Tempest tests in its +compliance test suite. However, it was never discussed if OVP should adopt this +policy as well. It turns out that this policy causes challenges for vendors of +commercial NFV offerings to pass the OVP test suite. The exemption process +outlined in this document aims at allowing to selectively disable strict API +response validation in order to enable vendors to adopt OVP **if** the tested +functionality is supported. + +It must be clearly understood that this exemption targets **only** the scenario +in which additional attributes are included in API responses. It does not +provide a loophole for passing OVP tests if the OpenStack APIs have been +altered significantly as this is in conflict with the objective of OVP to +create industry alignment. + +In conclusion, the exemption process described here is deemed beneficial for +both the broader industry as well as for OVP: Enabling adoption of OVP by +vendors which extended OpenStack API responses facilitates adoption of OVP in +the industry. The limited validity period of an exemption incentivizes eventual +alignment within the industry around a clearly specified set of APIs. + + +Example: additional attributes per VM for HA policy +--------------------------------------------------- + +This fictional example showcases the presence of an additional attribute in an +API response. The example shows that the 'server details', i.e. the VM +metadata, includes an additional attribute 'ha-policy' which is used to +associate high-availability policies with a VM instance. This attribute is +utilized by a proprietary add-on component to manage VM migration and recovery +in case of compute host failures:: + + { + "server": { + "accessIPv4": "1.2.3.4", + "config_drive": "", + "flavor": {...}, + "image": {...}, + "ha_policy": "migrate" <-- additional attribute + "name": "new-server-test", + "status": "ACTIVE" + } + } + + + +Precedent in OpenStack +====================== + +In the OpenStack community, the OpenStack Interoperability Working Group +(Interop WG) [4]_ is maintaining multiple API interoperability compliance +programs [5]_. These programs utilize Tempest-based tests to determine if a +given commercial cloud is compliant to a selected set of OpenStack APIs. After +introduction of the strict API response validation, various cloud products +which previously passed the compliance program failed validation because of the +reasons outlined above. + +In order to mitigate this situation, the Interop WG consulted with the broader +OpenStack community [6]_ and eventually introduced an "additional properties +waiver" for its API compliance programs in July 2016. The waiver was +created with a clearly defined validity period, covering roughly one year - +equivalent to three iterations of interoperability guidelines (2015.07, +2016.01, and 2016.08). The limited lifetime of the waiver was intended to give +cloud product vendors a transition period for adapting their products to +achieve full API compliance by the end of the exemption period. All details of +the waiver are listed in [7]_. Finally, the waiver was officially canceled in +October 2017 [8]_ after about 15 months. + + +Exemption process for additional properties in API responses in the OVP +======================================================================= + +The details of the exemption process for disabling strict validation of API +responses is as follows: + +#. The Dovetail tool provides a new command line option "--no-api-validation" for + disabling strict API validation. This option needs to be explicitly given on + the command line to disable strict API validation. If this command line + option is omitted, the default behavior (i.e., strict API validation) is + applied. + +#. The test results created by the Dovetail tool includes an explicit print-out + stating if strict API validation was disabled during the test run or not. + +#. The OVP portal reads the uploaded result files and indicates for all + uploaded test results if strict API validation was disabled or not. + +#. Together with the application for certification, a company can request an + exemption from the requirement for strict API response checking. A rationale + for requesting the exemption has to be provided. The request is either + granted or rejected by the C&C committee. The rationale provided must + establish that the need for exemption is not in violation of the OVP's + objectives. + +#. Compliance badges obtained under exemption are valid for one year. + +#. OPNFV expects OVP participants to aim for full compliance without requiring + exemptions as soon as possible. Hence, an exemption can only be requested + twice for the same product (addressing new versions of OVP or new versions + of the product). + +#. The same logo will be used regardless of being obtained under exemption or + not. + +#. The exemption will be made available to participants of OVP as part of a + service release of OVP 2018.01. + +#. The C&C committee will monitor the situation around exemptions and may + decide changes to the above process at any time, including the possibility + to stop issuing exemptions. + + +.. [1] https://review.openstack.org/#/c/156130/ +.. [2] https://github.com/openstack/tempest/tree/master/tempest/lib/api_schema/response/compute +.. [3] https://developer.openstack.org/api-ref/compute/#show-server-details +.. [4] https://wiki.openstack.org/wiki/Governance/InteropWG +.. [5] https://refstack.openstack.org/ +.. [6] http://lists.openstack.org/pipermail/openstack-dev/2016-June/097349.html +.. [7] https://review.openstack.org/#/c/333067/ +.. [8] https://review.openstack.org/#/c/512447/ diff --git a/docs/testing/user/cvpaddendum/index.rst b/docs/testing/user/ovpaddendum/index.rst index cd8a296a..054a3f11 100644 --- a/docs/testing/user/cvpaddendum/index.rst +++ b/docs/testing/user/ovpaddendum/index.rst @@ -3,10 +3,8 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) Intel and others -.. _dovetail-cvp_addendum: - ==================================================================== -Compliance Verification Program - Guidelines Addendum for Danube +OPNFV Verified Program - 2018.01 Guidelines Addendum for Danube ==================================================================== .. toctree:: @@ -17,12 +15,12 @@ Introduction ============ This addendum provides a high-level description of the testing scope and -pass/fail criteria used in the Compliance Verification Program (CVP) for the -OPNFV Danube release. This information is intended as an overview for CVP +pass/fail criteria used in the OPNFV Verified Program (OVP) for the +OPNFV Danube release. This information is intended as an overview for OVP testers and for the Dovetail Project to help guide test-tool and test-case development for the OPNFV Danube release. The Dovetail project is responsible for documenting -test-case specifications as well as implementing the CVP tool-chain through collaboration -with the OPNFV testing community. CVP testing focuses on establishing the +test-case specifications as well as implementing the OVP tool-chain through collaboration +with the OPNFV testing community. OVP testing focuses on establishing the ability of the System Under Test (SUT) to perform NFVI and VIM operations and support Service Provider oriented features that ensure manageable, resilient and secure networks. @@ -61,8 +59,8 @@ Assumptions about the System Under Test (SUT) include ... Scope of Testing ================ -The `OPNFV CVP Guidelines`_, as approved by the Board of Directors, outlines -the key objectives of the CVP as follows: +The `OVP Governance Guidelines`_, as approved by the Board of Directors, outlines +the key objectives of the OVP as follows: - Help build the market for @@ -87,8 +85,8 @@ the implementation of the underlying system under test". OPNFV provides a broad range of capabilities, including the reference platform itself as well as tools-chains and methodologies for building infrastructures, and deploying and testing the platform. -Not all these aspects are in scope for CVP and not all functions and -components are tested in the initial version of CVP. For example, the deployment tools +Not all these aspects are in scope for OVP and not all functions and +components are tested in the initial version of OVP. For example, the deployment tools for the SUT and CI/CD toolchain are currently out of scope. Similarly, performance benchmarking related testing is also out of scope or for further study. Newer functional areas such as MANO (outside of APIs in the NFVI and @@ -98,13 +96,13 @@ VIM) are still developing and are for future considerations. General Approach ---------------- -In order to meet the above objectives for CVP, we aim to follow a general approach +In order to meet the above objectives for OVP, we aim to follow a general approach by first identifying the overall requirements for all stake-holders, then analyzing what OPNFV and the upstream communities can effectively test and verify -presently to derive an initial working scope for CVP, and to recommend what the +presently to derive an initial working scope for OVP, and to recommend what the community should strive to achieve in future releases. -The overall requirements for CVP can be categorized by the basic cloud +The overall requirements for OVP can be categorized by the basic cloud capabilities representing common operations needed by basic VNFs, and additional requirements for VNFs that go beyond the common cloud capabilities including functional extensions, operational capabilities and additional carrier grade @@ -118,7 +116,7 @@ these basic requirements. We are not yet ready to include compliance requirements for capabilities such as hardware portability, carrier grade performance, fault management and other operational features, security, MANO and VNF verification. These areas are -being studied for consideration in future CVP releases. +being studied for consideration in future OVP releases. In some areas, we will start with a limited level of verification initially, constrained by what community resources are able to support at this @@ -144,7 +142,7 @@ In order to define the scope of the Danube-release of the compliance and verification program, this section analyzes NFV-focused platform capabilities with respect to the high-level objectives and the general approach outlined in the previous section. The analysis determines which capabilities are suitable -for inclusion in this release of the CVP and which capabilities are to be +for inclusion in this release of the OVP and which capabilities are to be addressed in future releases. 1. Basic Cloud Capabilities @@ -173,12 +171,12 @@ services. Running such basic VNF leads to a set of common requirements, includin - simple virtual machine resource scheduling on multiple nodes OPNFV mainly supports OpenStack as the VIM up to the Danube release. The -VNFs used in the CVP program, and features in scope for the program which are +VNFs used in the OVP program, and features in scope for the program which are considered to be basic to all VNFs, require commercial OpenStack distributions to support a common basic level of cloud capabilities, and to be compliant to a common specification for these capabilities. This requirement significantly overlaps with OpenStack community's Interop working group's goals, but they are not -identical. The CVP runs the OpenStack Refstack-Compute test cases to verify +identical. The OVP runs the OpenStack Refstack-Compute test cases to verify compliance to the basic common API requirements of cloud management functions and VNF (as a VM) management for OPNFV. Additional NFV specific requirements are added in network data path validation, @@ -199,12 +197,12 @@ NFV has functional requirements beyond the basic common cloud capabilities, esp. in the networking area. Examples like SDNVPN, IPv6, SFC may be considered additional NFV requirements beyond general purpose cloud computing. These feature requirements expand beyond common OpenStack (or other -VIM) requirements. OPNFV CVP will incorporate test cases to verify +VIM) requirements. OPNFV OVP will incorporate test cases to verify compliance in these areas as they become mature. Because these extensions may impose new API demands, maturity and industry adoption is a prerequisite for making them a mandatory requirement for OPNFV compliance. At the time of Danube, -we have not identified a new functional area that is mandatory for CVP. -In the meantime, CVP +we have not identified a new functional area that is mandatory for OVP. +In the meantime, OVP intends to offer tests in some of these areas as an optional extension of the test report to be submitted for review, noting that passing these tests will not be required to pass OPNFV compliance verification. @@ -231,7 +229,7 @@ and should be a mandatory requirement. The current test cases in HA cover the basic area of failure and resource overload conditions for a cloud platform's service availability, including all of the basic cloud capability services, and basic compute and storage loads, -so it is a meaningful first step for CVP. We expect additional high availability +so it is a meaningful first step for OVP. We expect additional high availability scenarios be extended in future releases. 4. Resiliency @@ -245,14 +243,14 @@ OPNFV system resiliency in the Danube release that can be used to provide limited coverage in this area. However, this is a relatively new test methodology in OPNFV, additional study and testing experiences are still needed. We defer the resiliency testing to -future CVP releases. +future OVP releases. 5. Security Security is among the top priorities as a carrier grade requirement by the end-users. Some of the basic common functions, including virtual network isolation, security groups, port security and role based access control are already covered as -part of the basic cloud capabilities that are verified in CVP. These test cases +part of the basic cloud capabilities that are verified in OVP. These test cases however do not yet cover the basic required security capabilities expected of an end-user deployment. It is an area that we should address in the near future, to define a common set of requirements and develop test cases for verifying those @@ -262,7 +260,7 @@ Another common requirement is security vulnerability scanning. While the OPNFV security project integrated tools for security vulnerability scanning, this has not been fully analyzed or exercised in Danube release. This area needs further work to identify the required level of security for the -purpose of OPNFV in order to be integrated into the CVP. End-user inputs on +purpose of OPNFV in order to be integrated into the OVP. End-user inputs on specific requirements in security is needed. 6. Service assurance @@ -293,38 +291,38 @@ NFVI features that users care about. There are a lot of projects in OPNFV developing use cases and sample VNFs, however most are still in early phase and require further enhancements to -become useful additions to the CVP. Examples such as vIMS, or those which are +become useful additions to the OVP. Examples such as vIMS, or those which are not yet available in Danube release, e.g. vCPE, will be valuable additions to -the CVP. These use cases need to be widely accepted, and since they are more -complex, using these VNFs for CVP demands a higher level of community resources +the OVP. These use cases need to be widely accepted, and since they are more +complex, using these VNFs for OVP demands a higher level of community resources to implement, analyze and document these VNFs. Hence, use case testing is not -ready for CVP at the time of Danube, but can be incorporated in Euphrates or as +ready for OVP at the time of Danube, but can be incorporated in Euphrates or as a future roadmap area. 8. Additional capabilities In addition to the capabilities analyzed above, there are further system -aspects which are of importance for the CVP. These comprise operational and +aspects which are of importance for the OVP. These comprise operational and management aspects such as platform in-place upgrades and platform operational insights such as telemetry and logging. Further aspects include API backward compatibility / micro-versioning, workload migration, multi-site federation and interoperability with workload automation platforms, e.g. ONAP. Finally, efficiency aspects such as the hardware and energy footprint of the platform -are worth considering in the CVP. +are worth considering in the OVP. OPNFV is addressing these items on different levels of details in different projects. However, the contributions developed in these projects are not yet considered widely available in commercial systems in order to include them in -the CVP. Hence, these aspects are left for inclusion in future releases of the -CVP. +the OVP. Hence, these aspects are left for inclusion in future releases of the +OVP. -Scope of the Danube-release of the CVP +Scope of the Danube-release of the OVP -------------------------------------- Summarizing the results of the analysis above, the scope of the Danube-release -of the CVP is as follows: +of the OVP is as follows: - Test Area: Basic cloud capabilities @@ -336,8 +334,8 @@ of the CVP is as follows: - *VM resource scheduling* - *Forwarding packets in the data path* -\* The OPNFV CVP utilizes the same set of test cases as the OpenStack -interoperability program *OpenStack Powered Compute*. Passing the OPNFV CVP +\* The OPNFV OVP utilizes the same set of test cases as the OpenStack +interoperability program *OpenStack Powered Compute*. Passing the OPNFV OVP does **not** imply that the SUT is certified according to the *OpenStack Powered Compute* program. *OpenStack Powered Compute* is a trademark of the OpenStack foundation and the corresponding certification label can only be awarded by the @@ -365,8 +363,8 @@ OpenStack foundation. These tested areas represent significant advancement in the direction to meet -the CVP's objectives and end-user expectations, and is a good basis for the -initial phase of CVP. +the OVP's objectives and end-user expectations, and is a good basis for the +initial phase of OVP. Note: The SUT is limited to NFVI and VIM functions. While testing MANO component capabilities is out of scope, certain APIs exposed towards MANO are @@ -375,7 +373,7 @@ elements may be part of the test infrastructure; for example used for workload deployment and provisioning. -Scope considerations for future CVP releases +Scope considerations for future OVP releases -------------------------------------------- Based on the previous analysis, the following items are outside the scope of @@ -416,7 +414,19 @@ documented and accepted by the reviewers. Applicants who choose to run the optional test cases can include the results of the optional test cases to highlight the additional compliance. + +Exemption from strict API response validation +============================================= + +Vendors of commercial NFVI products may have extended the Nova API to support +proprietary add-on features. These additions can cause Nova Tempest API tests +to fail due to unexpected data in API responses. In order to resolve this +transparently in the context of OVP, a temporary exemption process has been +created. More information on the exemption can be found in section +:ref:`dovetail-exemption_process_api_response_validation`. + + .. References -.. _`OPNFV CVP Guidelines`: https://wiki.opnfv.org/display/dovetail/CVP+document +.. _`OVP Governance Guidelines`: https://www.opnfv.org/wp-content/uploads/sites/12/2018/01/OVP-Governance-Guidelines-1.0.1-012218.pdf .. _`Pharos specification`: https://wiki.opnfv.org/display/pharos/Pharos+Specification diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_log_files.png b/docs/testing/user/reviewerguide/danube/images/ovp_log_files.png Binary files differnew file mode 100644 index 00000000..d5af068b --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_log_files.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_log_setup.png b/docs/testing/user/reviewerguide/danube/images/ovp_log_setup.png Binary files differnew file mode 100644 index 00000000..395a2c3e --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_log_setup.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_log_test_count.png b/docs/testing/user/reviewerguide/danube/images/ovp_log_test_count.png Binary files differnew file mode 100644 index 00000000..6c363441 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_log_test_count.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_missing_ha.png b/docs/testing/user/reviewerguide/danube/images/ovp_missing_ha.png Binary files differnew file mode 100644 index 00000000..7e2c6982 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_missing_ha.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_pass_fraction.png b/docs/testing/user/reviewerguide/danube/images/ovp_pass_fraction.png Binary files differnew file mode 100644 index 00000000..74c99f37 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_pass_fraction.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_pass_percentage.png b/docs/testing/user/reviewerguide/danube/images/ovp_pass_percentage.png Binary files differnew file mode 100644 index 00000000..ad79f4ec --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_pass_percentage.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_result_overview.png b/docs/testing/user/reviewerguide/danube/images/ovp_result_overview.png Binary files differnew file mode 100644 index 00000000..fa12ddc7 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_result_overview.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_result_review.png b/docs/testing/user/reviewerguide/danube/images/ovp_result_review.png Binary files differnew file mode 100644 index 00000000..a8f988fc --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_result_review.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_run_results.png b/docs/testing/user/reviewerguide/danube/images/ovp_run_results.png Binary files differnew file mode 100644 index 00000000..4b845df2 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_run_results.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_test_count.png b/docs/testing/user/reviewerguide/danube/images/ovp_test_count.png Binary files differnew file mode 100644 index 00000000..1c9bcd30 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_test_count.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_top_nav.png b/docs/testing/user/reviewerguide/danube/images/ovp_top_nav.png Binary files differnew file mode 100644 index 00000000..280d0c5f --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_top_nav.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_vping_ssh.png b/docs/testing/user/reviewerguide/danube/images/ovp_vping_ssh.png Binary files differnew file mode 100644 index 00000000..01384d02 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_vping_ssh.png diff --git a/docs/testing/user/reviewerguide/danube/images/ovp_vping_user.png b/docs/testing/user/reviewerguide/danube/images/ovp_vping_user.png Binary files differnew file mode 100644 index 00000000..a82e1121 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/ovp_vping_user.png diff --git a/docs/testing/user/reviewerguide/danube/images/sut_endpoints.png b/docs/testing/user/reviewerguide/danube/images/sut_endpoints.png Binary files differnew file mode 100644 index 00000000..f11d531f --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/sut_endpoints.png diff --git a/docs/testing/user/reviewerguide/danube/images/sut_info.png b/docs/testing/user/reviewerguide/danube/images/sut_info.png Binary files differnew file mode 100644 index 00000000..395f5f41 --- /dev/null +++ b/docs/testing/user/reviewerguide/danube/images/sut_info.png diff --git a/docs/testing/user/reviewerguide/index.rst b/docs/testing/user/reviewerguide/index.rst index fef318e9..bd615f19 100644 --- a/docs/testing/user/reviewerguide/index.rst +++ b/docs/testing/user/reviewerguide/index.rst @@ -2,9 +2,9 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) Ericsson AB -================================================== -Compliance and Verification program reviewer guide -================================================== +============================================= +OPNFV Verified Program 2018.01 Reviewer Guide +============================================= .. toctree:: :maxdepth: 2 @@ -13,12 +13,194 @@ Compliance and Verification program reviewer guide Introduction ============ -This reviewer guide provides detailed guidance for reviewers on how to handle the result review process. +This reviewer guide provides detailed guidance for reviewers on how to handle the result review +process. Reviewers must follow the checklist below to ensure review consistency for the OPNFV +Verified Program (OVP) 2018.01 (Danube) release at a minimum. -TODO: add sections on +#. **Mandatory Test Area Results** - Validate that results for all mandatory test areas are present. +#. **Test-Case Count within Mandatory Test Area** - Check that the total number of test-cases are present in each mandatory test area. +#. **Test-Case Pass Percentage** - Ensure all tests have passed (100% pass rate). +#. **Log File Verification** - Inspect the log file for each test area (osinterop, ha, vping). +#. **SUT Info Verification** - Validate the system under test (SUT) hardware and software endpoint info is present. - - specification of criteria for reviewing the results - - specification of criteria for granting the certification badge +1. Mandatory Test Area Results +============================== - - specification SLAs for responding to requests, questions, escalations, ... +Validate that results for all mandatory test areas are included in the overall test suite. The +required mandatory test areas are: + - **osinterop** + - **vping** + - **ha** + +Login to the OVP portal at: + +*https://verified.opnfv.org* + +Click on the 'My Results' tab in top-level navigation bar. + +.. image:: danube/images/ovp_top_nav.png + :align: center + :scale: 100% + +The OVP administrator will ask for review volunteers using the verified@opnfv.org email alias. The +incoming results for review will be identified by the administrator with particular 'Test ID' +and 'Owner' values. The corresponding OVP portal result will have a status of 'review'. + +.. image:: danube/images/ovp_result_review.png + :align: center + :scale: 100% + +In the example above, this information will be provided as: +- Test ID: a00c47e8 +- Owner: jtaylor + +Click on the hyperlink within the 'Test ID' column. + +*Note, that the 'Test ID' column in this view condenses the UUID used for 'Test ID' to +eight characters even though the 'Test ID' is a longer UUID in the back-end.* + +.. image:: danube/images/ovp_result_overview.png + :align: center + :scale: 100% + +The 'Test ID' hyperlink toggles the view to a top-level listing of the results displayed above. +Validate that osinterop, vping and ha test area results are all present within the view. + + +2. Test-Case Count within Mandatory Test Area +============================================= + +Validate the test-case count within each test area. For the OVP 2018.01 release, this must break +down as outlined in the table below. + +.. image:: danube/images/ovp_test_count.png + :align: center + :scale: 100% + +In the diagram above (from section 1), these counts can be gleaned from the numbers to the +right of the test-cases. The total number is given for the osinterop (dovetail.osinterop.tc001) +test area at 205. The vping (dovetail.vping.tc00x) and ha (dovetail.ha.tc00x) test-cases are +broken down separately with a line for each test-case. Directly above the 'Test Result Overview' +listing there's a summary labelled 'Test Run Results' shown below. For OVP 2018.01, a mandatory +total of **215** test-cases must be present (205 osinterop + 8 ha + 2 vping). + +.. image:: danube/images/ovp_missing_ha.png + :align: center + :scale: 100% + +An example of a listing that should flag a negative review is shown above. The mandatory ha test +area is missing one test case (dovetail.ha.tc008). + +3. Test-Case Pass Percentage +============================ + +All mandatory test-cases must pass. This can be validated in multiple ways. The below diagram of +the 'Test Run Results' is one method and shows that 100% of the mandatory test-cases have passed. +This value must not be lower than 100%. + +.. image:: danube/images/ovp_pass_percentage.png + :align: center + :width: 350 px + +Another method to check that all mandatory test-cases have passed is shown in the diagram below. +The pass/total is given as a fraction and highlighted here in yellow. For the osinterop test area, +the result must display [205/205] and for each of the test-cases under the vping and ha test areas +[1/1] must be displayed. + +.. image:: danube/images/ovp_pass_fraction.png + :align: center + :width: 270 px + +4. Log File Verification +======================== + +Three log files must be verified for content within each mandatory test area. The log files for +each of the test areas is noted in the table below. + +.. image:: danube/images/ovp_log_files.png + :align: center + :scale: 100% + +The three log files can be displayed by clicking on the setup icon to the right of the results, +as shown in the diagram below. + +*Note, while the vping and ha test areas list multiple test-cases in the below diagram, there is +a single log file for all test-cases within these test areas.* + +.. image:: danube/images/ovp_log_setup.png + :align: center + :scale: 100% + +Within the osinterop log (dovetail.osinterop.tc001.log), scroll down to the area of the log that +begins to list the results of each test-case executed. This can be located by looking for lines +prefaced with '**tempest.api**' and ending with '**... ok**'. + +.. image:: danube/images/ovp_log_test_count.png + :align: center + :scale: 100% + +The number of lines within the osinterop log for test-cases must add up according to the table +above, where test-cases are broken down according to compute, identity, image, network and volume, +with respective counts given in the table. The ha log (yardstick.log) must contain the 'PASS' +result for each of the eight test-cases within this test area. This can be verified by searching +the log for the keyword 'PASS'. + + + +The eight lines to validate are listed below: + + - 017-10-16 05:07:49,158 yardstick.benchmark.scenarios.availability.serviceha serviceha.py:81 + INFO The HA test case PASS the SLA + - 2017-10-16 05:08:31,387 yardstick.benchmark.scenarios.availability.serviceha serviceha.py:81 + INFO The HA test case PASS the SLA + - 2017-10-16 05:09:13,669 yardstick.benchmark.scenarios.availability.serviceha serviceha.py:81 + INFO The HA test case PASS the SLA + - 2017-10-16 05:09:55,967 yardstick.benchmark.scenarios.availability.serviceha serviceha.py:81 + INFO The HA test case PASS the SLA + - 2017-10-16 05:10:38,407 yardstick.benchmark.scenarios.availability.serviceha serviceha.py:81 + INFO The HA test case PASS the SLA + - 2017-10-16 05:11:00,030 yardstick.benchmark.scenarios.availability.scenario_general + scenario_general.py:71 INFO [92m Congratulations, the HA test case PASS! [0m + - 2017-10-16 05:11:22,536 yardstick.benchmark.scenarios.availability.scenario_general + scenario_general.py:71 INFO [92m Congratulations, the HA test case PASS! [0m + - 2017-10-16 05:12:07,880 yardstick.benchmark.scenarios.availability.scenario_general + scenario_general.py:71 INFO [92m Congratulations, the HA test case PASS! [0m + + +The final validation is for the vping test area log file (functest.log). The two entries +displayed in the diagrams below must be present in this log file. + + - vping_userdata + - vping_ssh + +.. image:: danube/images/ovp_vping_user.png + :align: center + :scale: 100% + +.. image:: danube/images/ovp_vping_ssh.png + :align: center + :scale: 100% + +5. SUT Info Verification +======================== + +SUT information must be present in the results to validate that all required endpoint services +and at least two controllers were present during test execution. For the results shown below, +click the '**info**' hyperlink in the **SUT** column to navigate to the SUT information page. + +.. image:: danube/images/sut_info.png + :align: center + :scale: 100% + +In the '**Endpoints**' listing shown below for the SUT VIM component, ensure that services are +present for identify, compute, image, volume and network at a minimum by inspecting the +'**Service Type**' column. + +.. image:: danube/images/sut_endpoints.png + :align: center + :scale: 100% + +Inspect the '**Hosts**' listing found below the Endpoints secion of the SUT info page and ensure +at least two hosts are present, as two controllers are required the for the mandatory HA +test-cases. diff --git a/docs/testing/user/systempreparation/index.rst b/docs/testing/user/systempreparation/index.rst index fe1f60d6..afc5dc47 100644 --- a/docs/testing/user/systempreparation/index.rst +++ b/docs/testing/user/systempreparation/index.rst @@ -5,25 +5,25 @@ .. _dovetail-system_preparation_guide: -============================================================ -Compliance Verification Program system preparation guide -============================================================ +=============================================== +OPNFV Verified Program system preparation guide +=============================================== This document provides a general guide to hardware system prerequisites -and expectations for running OPNFV CVP testing. For detailed guide of +and expectations for running OPNFV OVP testing. For detailed guide of preparing software tools and configurations, and conducting the test, please refer to the User Guide :ref:dovetail-testing_user_guide. -The CVP test tools expect that the hardware of the System Under Test (SUT) +The OVP test tools expect that the hardware of the System Under Test (SUT) is Pharos compliant `Pharos specification`_ The Pharos specification itself is a general guideline, rather than a set of specific hard requirements at this time, developed by the OPNFV community. For -the purpose of helping CVP testers, we summarize the main aspects of hardware to -consider in preparation for CVP testing. +the purpose of helping OVP testers, we summarize the main aspects of hardware to +consider in preparation for OVP testing. -As described by the CVP Testing User Guide, the hardware systems involved in -CVP testing includes a Test Node, a System Under Test (SUT) system, and network +As described by the OVP Testing User Guide, the hardware systems involved in +OVP testing includes a Test Node, a System Under Test (SUT) system, and network connectivity between them. The Test Node can be a bare metal machine or a virtual machine that can support @@ -39,13 +39,13 @@ ARM-64. Mixing different architectures in the same SUT is not supported. A minimum of 5 servers, 3 configured for controllers and 2 or more configured for compute resource are expected. However this is not a hard requirement -at this phase. The CVP 1.0 mandatory test cases only require one compute server. At +at this phase. The OVP 1.0 mandatory test cases only require one compute server. At lease two compute servers are required to pass some of the optional test cases -in the current CVP release. CVP control service high availability tests expect two +in the current OVP release. OVP control service high availability tests expect two or more control nodes to pass, depending on the HA mechanism implemented by the SUT. -The SUT is also expected to include components for persistent storage. The CVP +The SUT is also expected to include components for persistent storage. The OVP testing does not expect or impose significant storage size or performance requirements. The SUT is expected to be connected with high performance networks. These networks @@ -56,7 +56,7 @@ and compute services in the SUT - A data network that supports the virtual network capabilities and data path testing Additional networks, such as Light Out Management or storage networks, may be -beneficial and found in the SUT, but they are not a requirement for CVP testing. +beneficial and found in the SUT, but they are not a requirement for OVP testing. .. References .. _`Pharos specification`: https://wiki.opnfv.org/display/pharos/Pharos+Specification diff --git a/docs/testing/user/testspecification/dynamicnetwork/index.rst b/docs/testing/user/testspecification/dynamicnetwork/index.rst index cda1d5b6..a25e4f66 100644 --- a/docs/testing/user/testspecification/dynamicnetwork/index.rst +++ b/docs/testing/user/testspecification/dynamicnetwork/index.rst @@ -59,7 +59,7 @@ a previous test. Specifically, every test performs clean-up operations which return the system to the same state as before the test. All these test cases are included in the test case dovetail.tempest.tc003 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/forwardingpackets/index.rst b/docs/testing/user/testspecification/forwardingpackets/index.rst index c12e0939..4a4ffea9 100644 --- a/docs/testing/user/testspecification/forwardingpackets/index.rst +++ b/docs/testing/user/testspecification/forwardingpackets/index.rst @@ -49,7 +49,7 @@ The test area is structured based on the basic operations of forwarding packets in data path through virtual networks. Specifically, the test performs clean-up operations which return the system to the same state as before the test. -This test case is included in the test case dovetail.tempest.tc001 of cvp test suite. +This test case is included in the test case dovetail.tempest.tc001 of OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/index.rst b/docs/testing/user/testspecification/index.rst index 858d02ce..2a8298c9 100644 --- a/docs/testing/user/testspecification/index.rst +++ b/docs/testing/user/testspecification/index.rst @@ -5,23 +5,23 @@ .. _dovetail-test_case_specification: ================================================== -Compliance Verification program test specification +OPNFV Verified Program test specification ================================================== Introduction ============ -The OPNFV CVP provides a series or test areas aimed to evaluate the operation +The OPNFV OVP provides a series or test areas aimed to evaluate the operation of an NFV system in accordance with carrier networking needs. Each test area contains a number of associated test cases which are described in detail in the associated test specification. -All tests in the CVP are required to fulfill a specific set of criteria in -order that the CVP is able to provide a fair assessment of the system under +All tests in the OVP are required to fulfill a specific set of criteria in +order that the OVP is able to provide a fair assessment of the system under test. Test requirements are described in the 'Test Case Requirements'_ document. -All tests areas addressed in the CVP are covered in the following test +All tests areas addressed in the OVP are covered in the following test specification documents. .. toctree:: diff --git a/docs/testing/user/testspecification/machinelifecycle/index.rst b/docs/testing/user/testspecification/machinelifecycle/index.rst index 5b46b90f..b0cc0d79 100644 --- a/docs/testing/user/testspecification/machinelifecycle/index.rst +++ b/docs/testing/user/testspecification/machinelifecycle/index.rst @@ -60,7 +60,7 @@ created by a previous test. Specifically, every test performs clean-up operations which return the system to the same state as before the test. All these test cases are included in the test case dovetail.tempest.tc004 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/multiplenodes/index.rst b/docs/testing/user/testspecification/multiplenodes/index.rst index 2049f601..4b4859a9 100644 --- a/docs/testing/user/testspecification/multiplenodes/index.rst +++ b/docs/testing/user/testspecification/multiplenodes/index.rst @@ -54,7 +54,7 @@ the state created by a previous test. Specifically, every test performs clean-up operations which return the system to the same state as before the test. All these test cases are included in the test case dovetail.tempest.tc005 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/securitygroup/index.rst b/docs/testing/user/testspecification/securitygroup/index.rst index daacd963..61aa1c4b 100644 --- a/docs/testing/user/testspecification/securitygroup/index.rst +++ b/docs/testing/user/testspecification/securitygroup/index.rst @@ -54,7 +54,7 @@ the state created by a previous test. Specifically, every test performs clean-up operations which return the system to the same state as before the test. All these test cases are included in the test case dovetail.tempest.tc002 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/vimoperationscompute/index.rst b/docs/testing/user/testspecification/vimoperationscompute/index.rst index ea5441ac..64f4356b 100644 --- a/docs/testing/user/testspecification/vimoperationscompute/index.rst +++ b/docs/testing/user/testspecification/vimoperationscompute/index.rst @@ -69,7 +69,7 @@ For brevity, the test cases in this test area are summarized together based on the operations they are testing. All these test cases are included in the test case dovetail.osinterop.tc001 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/vimoperationsidentity/index.rst b/docs/testing/user/testspecification/vimoperationsidentity/index.rst index c2282f85..9790e75e 100644 --- a/docs/testing/user/testspecification/vimoperationsidentity/index.rst +++ b/docs/testing/user/testspecification/vimoperationsidentity/index.rst @@ -54,7 +54,7 @@ The test area is structured based on VIM identity operations. Each test case is able to run independently, i.e. irrelevant of the state created by a previous test. All these test cases are included in the test case dovetail.osinterop.tc001 of -cvp test suite. +OVP test suite. Dependency Description ====================== diff --git a/docs/testing/user/testspecification/vimoperationsimage/index.rst b/docs/testing/user/testspecification/vimoperationsimage/index.rst index 6bf3f4c8..2970207e 100644 --- a/docs/testing/user/testspecification/vimoperationsimage/index.rst +++ b/docs/testing/user/testspecification/vimoperationsimage/index.rst @@ -55,7 +55,7 @@ For brevity, the test cases in this test area are summarized together based on the operations they are testing. All these test cases are included in the test case dovetail.osinterop.tc001 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/vimoperationsnetwork/index.rst b/docs/testing/user/testspecification/vimoperationsnetwork/index.rst index 72ad5766..bf929828 100644 --- a/docs/testing/user/testspecification/vimoperationsnetwork/index.rst +++ b/docs/testing/user/testspecification/vimoperationsnetwork/index.rst @@ -57,7 +57,7 @@ For brevity, the test cases in this test area are summarized together based on the operations they are testing. All these test cases are included in the test case dovetail.osinterop.tc001 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/testspecification/vimoperationsvolume/index.rst b/docs/testing/user/testspecification/vimoperationsvolume/index.rst index c59deb2d..ce51b954 100644 --- a/docs/testing/user/testspecification/vimoperationsvolume/index.rst +++ b/docs/testing/user/testspecification/vimoperationsvolume/index.rst @@ -70,7 +70,7 @@ For brevity, the test cases in this test area are summarized together based on the operations they are testing. All these test cases are included in the test case dovetail.osinterop.tc001 of -cvp test suite. +OVP test suite. Test Descriptions ================= diff --git a/docs/testing/user/teststrategy/index.rst b/docs/testing/user/teststrategy/index.rst deleted file mode 100644 index db05e035..00000000 --- a/docs/testing/user/teststrategy/index.rst +++ /dev/null @@ -1,301 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) Ericsson AB - -====================================================== -Compliance and Verification program test specification -====================================================== - -.. toctree:: - :maxdepth: 2 - -Version history -=============== - -+------------+----------+------------------+----------------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+----------------------------------+ -| 2017-03-15 | 0.0.1 | Chris Price | Draft version | -| | | | | -+------------+----------+------------------+----------------------------------+ - - -Introduction -============ - -This test specification provides a detailed outline of the prerequisites for performing evaluation -testing, testing methods and procedures used in the evaluation testing process and the tools provided -for running OPNFV evaluation testing. - -A CVP system under test is assumed to be a stand alone cloud infrastructure running a virtualisation -software stack providing high availability, redundancy and resiliency. OPNFV CVP testing covers the -combination of hardware and software that provide an NFV platform for running virtual workloads, approximately -the VIM & VNFi as defined by ETSI NFV ISG. - -While the NFV platform is a composite system under test comprised of both hardware and software the VIM -& NFVi testing focuses on software evaluation where it is required and assumed the software is running on -a platform deemed to be Pharos compliant. A "Pharos compliant" stand alone hardware system can be summarised -as a POD of at least 3 control and two compute blades to exercise the minimum set of compliance testing. -Pharos compliance is further defined, and expected to be implemented according to the `"Pharos specification"`_. - - -------- -Purpose -------- - -This document is intended to be read by an engineer, intending to run or prepare a system for the evaluation -tests, prior to beginning the preparation for executing the evaluation tests. The document is also useful as -a reference to learn more about OPNFV CVP testing, it's assumptions, targets, methods & tools and expected outcomes. - -The engineer will be guided through configuring and populating an environment that is suitable for executing the -OPNFV compliance evaluation test suite. This includes interpretations of the Pharos specification and assumptions -made by the toolchain on how those interpretations are evaluated. - -In addition to system preparation the document provides a guide for working with the methods and tools associated -with the evaluation test procedure. - ----------------- -Scope of testing ----------------- - -The OPNFV CVP testing suite is implemented to evaluate the compliance of a virtualisation platform with standard -NFV, carrier and communications network, platform requirements. Testing focuses on evaluating the software layer -in the platform in it's ability to provide OPNFV features and behaviours required to host communication and networking -industry applications. This section will provide a brief overview of the target areas and scope addressed by the -testing suites - -Features --------- - -CVP testing addresses primarily features and capabilities of value to, and further developed by, the OPNFV community. -Target areas for CVP testing are to verify the presence and compliance of: - * Validation of common cloud environment requirements inherited from key communities such as OpenStack and ETSI - * Capabilities required to provide consistent application and workload on-boarding, lifecycle management, and scaling - * Networking features required to address a carrier environment including VPN, service chaining, Trunking and peering - * Others?? - -Resilience ----------- - -Availability of service and infrastructure are fundamental principals of carrier networking and as such form a -component of the compliance test suite aimed to establish the ability of a virtualisation infrastructure to identify -and recover from failures across the system. The evaluation criteria specifically target control plane resilience -and the ability to identify, accommodate and recover from hardware failures in the system. - -Security? ---------- - -https://jira.opnfv.org/browse/DOVETAIL-382 - -This section should outline the test strategy related to security, -representing the test cases and requirements on the SUT for security testing. - -Scale ------ - -The ability to scale is important for carrier networking infrastructure and applications. The first iteration of the -compliance evaluation test suites address the need to scale however do not enforce strict requirements on how scale is -achieved. - -Compliance to the Pharos specification is evaluated as a component of the test suite and provides an visibility into -the physical infrastructures ability to scale in accordance with OPNFV requirements. The test suite itself does not -require an infrastructure that is running to be deployed at scale in order to pass the tests. It is assumed the -compliance to Pharos provides enough infrastructure validation that the capability is inherent. - -Characteristics ---------------- - -The OPNFV community invests heavily in ensuring the features and capabilities of the stack are able to run in the -most performant manner according to the needs of the workloads. This can range from the ability to linearly scale -workloads to the ability to process traffic at line rates. - -While each of these is a critical factor in evaluating the performance of a virtualisation infrastructure the CVP -suite does not at this time specify strict requirements on any given capability as part of the evaluation. It is -expected that in future test suites concise performance metrics will be required to be achieved to achieve compliance -at this time the community has elected not to place pass/fail requirements on characteristics. - - ---------------------- -Definitions and terms ---------------------- - -This document uses a number of acronyms and terms the reader may not be familiar with. For a full glossary of -terms used in OPNFV, refer to the `OPNFV Glossary`_. - -+------------+----------------------------------------------------------------+ -| **Term** | **Description** | -| | | -+------------+----------------------------------------------------------------+ -| CVP | The OPNFVCompliance and Verification Program | -| | | -+------------+----------------------------------------------------------------+ -| SUT | System under test; the complete system targeted by the | -| | test cases, including software hardware and configuration. | -| | | -+------------+----------------------------------------------------------------+ -| More | Additional entries to be added to this table | -| | | -+------------+----------------------------------------------------------------+ - - -Overview -======== - -This section of the document will describe in details the processes and procedures required to perform OPNFV CVP -compliance testing. The section is structured to address; planning and preparation, the approach to testing, the -scope of test activities including details of the test areas, methods and tools used in the testing and the result, -reporting & output of the test suites when run. - -Test planning and preparation -============================= - -https://jira.opnfv.org/browse/DOVETAIL-384 - -Give an outline of the planning phase, refer to the detailed system prep guide and test guides here. - - ../../systempreparation/index.rst - -Feature testing scope and approach -================================== - -------------------- -Pre-test validation -------------------- - -Describe how to evaluate test readiness here. -I suggest this be a process of doing a "dry run" and evaluating the results on the DB. This should not -need to be reproduced later in the document. - - -------------- -Test approach -------------- - -Here we should describe the way we approach testing different areas. API through RefStack, resilience through -ETSI test implementations, security is done in xyz way. This should serve as an introduction to the following -feature test scope sections and provide common information not to be replicated further down. - - ------------------- -Feature test scope ------------------- - -Included test areas -------------------- - -CVP testing for the Danube release focuses on evaluating the ability of a platform to host basic carrier -networking related workloads. The testing focuses on establishing the ability of the SUT to perform -basic NFVi operations such as managing images, instatiating workload & networking these workloads in a -secure and resiliant manner. - -Many OPNFV features are derived from our target upstream communities resulting in the tests focusing -on exposing behaviour present through those development efforts. This approach to OPNFV development -is reflected in our CVP testing where upstream validation procedures are leveraged to validate the -composite platform. The OpenStack `RefStack <https://refstack.openstack.org/#/>`_ test suites are -leveraged in OPNFV for performing VIM validation according to expected behaviour in OPNFV. - -OPNFV CVP testing has explicit requirements on hardware, control plane and compute topologies which -are assumed to be in place, these are covered in the `Preparing the virtualisation infrastructure`_ -section of this document. Tests may fail if the system is not prepared and configured in accordance -the prpeparation guidelines. - -Excluded test areas -------------------- - -The CVP testing procedure in Danube do not cover all aspects of the available OPNFV system, nor feature -suites. To ensure the highest quality of evaluation that provides a trustworthy foundation for industry -compliance toward a common OPNFV standard tests and test areas are selected based on three key principals; -maturity of the test area and framework, availability of features and capabilities across OPNFV compositions, -relevance and value to the industry for evaluation. - -In the Danube release of the CVP we have elected to esbalish an evaluation suite for only the common base -features of the platform. Features areas that are optional or not yet mature in the platform are expluded. -This includes a number of optinal networking features such as BGP VPN networking and Service Chaining which -are intended to be included as optional evaluation areas in future CVP releases. - -The Danube release of the OPNFV CVP testing suite in addition dose not attempt to provide an evaluation criteria -for hardware. Any hardware used in the evaluation testing must comply to the pre-resuities outlined in the -`Preparing the virtualisation infrastructure`_ section of this document. Although the hardware is not tested -itself and no qualification metric has been established for the hardware in the Danube release. - -Test criteria and reporting ---------------------------- - -https://jira.opnfv.org/browse/DOVETAIL-389 - -This section should specify the criteria to be used to decide whether a test item has passed or failed. -As each area may have differences it is important to ensure the user can react to a failure in accordance -with it's relevance to the overall test suites running. - -Critical functions that are mandatory should be identified here. If any of these fail the system testing -should be halted. If we can having a grading or sorts described here would be helpful as a "guide" for a -tester to know if they should submit, resolve an item in their stack, or try again at another time. - - ---------------------- -Test design and tools ---------------------- - -This section needs to be done once we know the tools and areas covered by the CVP testing suites. Parked for now. - -VIM NBI testing ---------------- - -Describe the test areas addressed and how the tools are designed. It is important to understand the behaviour -of the testing framework when running these tests. Here we get into the details of behaviour for each of -the test frameworks we use, what they are testing and how the results are to be interpreted. - -Outline the tool in detail, in this case RefStack. How does it work, is it run to completion, is reporting -done per test case when do I as a user know I should do something? - -Summarise the tests to be executed by this test framework and the purpose of those tests in the evaluation -of the CVP. Are there dependancies between tests, does the tool expect a certain behaviour, do the test -cases have specific dependancies. This provides the overall context of the evaluation performed by this -toolchain / suite and I would not want to be surprised by something when I run the tests after reading this. - -Next test area --------------- - -What is the test area, what tools, how do they work what does it mean to me as a tester? - -Another test area ------------------ - -Again what is the test area, what tools, how do they work what does it mean to me as a tester? - - -CVP test execution -================== - -This section should identify the test procedure being executed. Include all people and -roles involved in the execution of a CVP test. Include procedures, mailing lists, escalation, support -and periods of waiting for results. - -The general workflow I assume should be something like this: - -* Log into the CVP test case staging computer -* Open a Browser and Log into the CVP tool. -* Select the CVP suite to run, agree to the questions (there should be only 1 for now) -* Run the tests (we should be able to launch this from the Web UI if they are on the hosting machine) -* Wait for the tool to report completed. -* Review your results, refer to trouble shooting or support as needed -* Submit your results for CVP evaluation - ------------- -Test reports ------------- - -Describe the process of producing and accessing the test report. This shoudl be a sub-section of CVP test execution I think. - -how do I connect a test suite to my account to get a report? How do I access the report when it is ready, -how do I identify one report from another in the toolchain? We should go into all the details here and point -to the tool, referring to the "preparation" section of this document if needed for context. - - -References -========== - -.. _`"Pharos specification"`: https://opnfv.org/ -.. _`OPNFV Glossary`: http://docs.opnfv.org/en/latest/glossary/index.html - diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 89e76c11..70f78b5c 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -5,7 +5,7 @@ .. _dovetail-testing_user_guide: ******************************************************** -Compliance Verification Program Testing User Guide +OPNFV Verified Program Testing User Guide ******************************************************** .. toctree:: diff --git a/docs/testing/user/userguide/testing_guide.rst b/docs/testing/user/userguide/testing_guide.rst index e55c1595..3fc74028 100644 --- a/docs/testing/user/userguide/testing_guide.rst +++ b/docs/testing/user/userguide/testing_guide.rst @@ -3,15 +3,15 @@ .. (c) OPNFV, Huawei Technologies Co.,Ltd and others. ========================================== -Conducting CVP Testing with Dovetail +Conducting OVP Testing with Dovetail ========================================== Overview ------------------------------ -The Dovetail testing framework for CVP consists of two major parts: the testing client that +The Dovetail testing framework for OVP consists of two major parts: the testing client that executes all test cases in a lab (vendor self-testing or a third party lab), -and the server system that is hosted by the CVP administrator to store and +and the server system that is hosted by the OVP administrator to store and view test results based on a web API. The following diagram illustrates this overall framework. @@ -26,7 +26,7 @@ The above diagram assumes that the tester's Test Host is situated in a DMZ, whic has internal network access to the SUT and external access via the public Internet. The public Internet connection allows for easy installation of the Dovetail containers. A singular compressed file that includes all the underlying results can be pulled from -the Test Host and uploaded to the OPNFV CVP server. +the Test Host and uploaded to the OPNFV OVP server. This arrangement may not be supported in some labs. Dovetail also supports an offline mode of installation that is illustrated in the next diagram. @@ -41,11 +41,11 @@ the Test Host. While it is possible to run the Test Host as a virtual machine, this user guide assumes it is a physical machine for simplicity. The rest of this guide will describe how to install the Dovetail tool as a -Docker container image, go over the steps of running the CVP test suite, and +Docker container image, go over the steps of running the OVP test suite, and then discuss how to view test results and make sense of them. Readers interested -in using Dovetail for its functionalities beyond CVP testing, e.g. for in-house +in using Dovetail for its functionalities beyond OVP testing, e.g. for in-house or extended testing, should consult the Dovetail developer's guide for additional information. @@ -109,7 +109,7 @@ Installing Prerequisite Packages on the Test Host The main prerequisite software for Dovetail are Python and Docker. -In the CVP test suite for the Danube release, Dovetail requires Python 2.7. Various minor +In the OVP test suite for the Danube release, Dovetail requires Python 2.7. Various minor versions of Python 2.7.x are known to work Dovetail, but there are no assurances. Python 3.x is not supported at this time. @@ -375,7 +375,7 @@ Installing Dovetail on the Test Host The Dovetail project maintains a Docker image that has Dovetail test tools preinstalled. This Docker image is tagged with versions. Before pulling the Dovetail image, check the -OPNFV's CVP web page first to determine the right tag for CVP testing. +OPNFV's OVP web page first to determine the right tag for OVP testing. Online Test Host """""""""""""""" @@ -488,10 +488,10 @@ Build Local DB and Testapi Services ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The steps in this section only need to be executed if the user plans on storing consolidated -results on the Test Host that can be uploaded to the CVP portal. +results on the Test Host that can be uploaded to the OVP portal. Dovetail needs to build the local DB and testapi service for storing and reporting results -to the CVP web portal. There is a script in the Dovetail container for building the local DB. +to the OVP web portal. There is a script in the Dovetail container for building the local DB. The ports 27017 and 8000 are used by the DB and testapi respectively. If the Test Host is using these ports for existing services, to avoid conflicts, remap the ports to values that are unused. Execute the commands below in the Dovetail container to remap ports, as required. @@ -515,8 +515,8 @@ To validate the DB and testapi services are running successfully, navigate to th the IP address of the Test Host and the testapi port number. If you can access this URL successfully, the services are up and running. -Running the CVP Test Suite --------------------------- +Running the OVP Test Suite +---------------------------- All or a subset of the available tests can be executed at any location within the Dovetail container prompt. You can refer to :ref:`cli-reference` @@ -528,7 +528,7 @@ for the details of the CLI. $ dovetail run --testsuite <test-suite-name> The '--testsuite' option is used to control the set of tests intended for execution -at a high level. For the purposes of running the CVP test suite, the test suite name follows +at a high level. For the purposes of running the OVP test suite, the test suite name follows the following format, ``ovp.<major>.<minor>.<patch>``. The latest and default test suite is ovp.1.0.0. @@ -553,9 +553,20 @@ arguments 'ipv6', 'sdnvpn' and 'tempest'. $ dovetail run --testarea mandatory +Dovetail allows the user to disable strict API response validation implemented +by Nova Tempest tests by means of the ``--no-api-validation`` option. Usage of +this option is only advisable if the SUT returns Nova API responses that +contain additional attributes. For more information on this command line option +and its intended usage, refer to +:ref:`dovetail-exemption_process_api_response_validation`. + +.. code-block:: bash + + $ dovetail run --no-api-validation + By default, results are stored in local files on the Test Host at ``$DOVETAIL_HOME/results``. Each time the 'dovetail run' command is executed, the results in the aforementioned directory -are overwritten. To create a singular compressed result file for upload to the CVP portal or +are overwritten. To create a singular compressed result file for upload to the OVP portal or for archival purposes, the results need to pushed to the local DB. This can be achieved by using the '--report' option with an argument syntax as shown below. Note, that the Test Host IP address and testapi port number must be substituted with appropriate values. @@ -594,10 +605,10 @@ When test execution is complete, a tar file with all result and log files is wri ``$DOVETAIL_HOME`` on the Test Host. An example filename is ``${DOVETAIL_HOME}/logs_20180105_0858.tar.gz``. The file is named using a timestamp that follows the convention 'YearMonthDay-HourMinute'. In this case, it was generated -at 08:58 on January 5th, 2018. This tar file is used to upload to the CVP portal. +at 08:58 on January 5th, 2018. This tar file is used to upload to the OVP portal. -Making Sense of CVP Test Results +Making Sense of OVP Test Results ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When a tester is performing trial runs, Dovetail stores results in local files on the Test @@ -646,10 +657,10 @@ and dovetail_ha_tcXXX.out will not be created. ``sdnvpn_logs/dovetail.sdnvpn.tcXXX.log`` and ``tempest_logs/dovetail.tempest.tcXXX.log``, respectively. They all have the passed, skipped and failed test cases results. -CVP Portal Web Interface +OVP Portal Web Interface ------------------------ -The CVP portal is a public web interface for the community to collaborate on results +The OVP portal is a public web interface for the community to collaborate on results and to submit results for official OPNFV compliance verification. The portal can be used as a resource by users and testers to navigate and inspect results more easily than by manually inspecting the log files. The portal also allows users to share results in a private manner @@ -699,7 +710,7 @@ Updating Dovetail or a Test Suite --------------------------------- Follow the instructions in section `Installing Dovetail on the Test Host`_ and -`Running the CVP Test Suite`_ by replacing the docker images with new_tags, +`Running the OVP Test Suite`_ by replacing the docker images with new_tags, .. code-block:: bash @@ -707,6 +718,6 @@ Follow the instructions in section `Installing Dovetail on the Test Host`_ and sudo docker pull opnfv/functest:<functest_new_tag> sudo docker pull opnfv/yardstick:<yardstick_new_tag> -This step is necessary if dovetail software or the CVP test suite have updates. +This step is necessary if dovetail software or the OVP test suite have updates. diff --git a/dovetail/compliance/cvp.0.8.0.yml b/dovetail/compliance/cvp.0.8.0.yml deleted file mode 100644 index 942215b9..00000000 --- a/dovetail/compliance/cvp.0.8.0.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -cvp.0.8.0: - name: cvp.0.8.0 - testcases_list: - # mandatory test cases - # osinterop - - dovetail.osinterop.tc001 - # vping - - dovetail.vping.tc001 - - dovetail.vping.tc002 - # HA - - dovetail.ha.tc001 - - dovetail.ha.tc002 - - dovetail.ha.tc003 - - dovetail.ha.tc004 - - dovetail.ha.tc005 - - dovetail.ha.tc006 - - dovetail.ha.tc007 - - dovetail.ha.tc008 - # optional test cases - # ipv6 - - dovetail.ipv6.tc001 - - dovetail.ipv6.tc002 - - dovetail.ipv6.tc003 - - dovetail.ipv6.tc004 - - dovetail.ipv6.tc005 - - dovetail.ipv6.tc006 - - dovetail.ipv6.tc007 - - dovetail.ipv6.tc008 - - dovetail.ipv6.tc009 - - dovetail.ipv6.tc010 - - dovetail.ipv6.tc011 - - dovetail.ipv6.tc012 - - dovetail.ipv6.tc013 - - dovetail.ipv6.tc014 - - dovetail.ipv6.tc015 - - dovetail.ipv6.tc016 - - dovetail.ipv6.tc017 - - dovetail.ipv6.tc018 - - dovetail.ipv6.tc019 - - dovetail.ipv6.tc020 - - dovetail.ipv6.tc021 - - dovetail.ipv6.tc022 - - dovetail.ipv6.tc023 - - dovetail.ipv6.tc024 - - dovetail.ipv6.tc025 - # tempest - - dovetail.tempest.tc001 - - dovetail.tempest.tc002 - - dovetail.tempest.tc003 - - dovetail.tempest.tc004 - - dovetail.tempest.tc005 - # sdnvpn - - dovetail.sdnvpn.tc001 - - dovetail.sdnvpn.tc002 - - dovetail.sdnvpn.tc004 - - dovetail.sdnvpn.tc008 diff --git a/dovetail/compliance/cvp.0.9.0.yml b/dovetail/compliance/ovp.1.0.0.yml index 8b9da9e1..71f2198d 100644 --- a/dovetail/compliance/cvp.0.9.0.yml +++ b/dovetail/compliance/ovp.1.0.0.yml @@ -1,6 +1,6 @@ --- -cvp.0.9.0: - name: cvp.0.9.0 +ovp.1.0.0: + name: ovp.1.0.0 testcases_list: # mandatory test cases # osinterop diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index 957a279f..2fd9834b 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -28,19 +28,19 @@ cli: - '-f' path: - 'functest/docker_tag' - help: 'Overwrite tag for functest docker container (e.g. cvp.0.5.0)' - bott_tag: - flags: - - '--bott_tag' - - '-b' - path: - - 'bottlenecks/docker_tag' - help: 'Overwrite tag for bottlenecks docker container (e.g. cvp.0.4.0)' + help: 'Overwrite tag for functest docker container (e.g. ovp.1.0.0)' + # bott_tag: + # flags: + # - '--bott_tag' + # - '-b' + # path: + # - 'bottlenecks/docker_tag' + # help: 'Overwrite tag for bottlenecks docker container (e.g. cvp.0.4.0)' control: testsuite: flags: - '--testsuite' - default: 'cvp.0.9.0' + default: 'ovp.1.0.0' help: 'compliance testsuite.' testarea: flags: @@ -63,3 +63,8 @@ cli: - '--offline' is_flag: 'True' help: 'run in offline method, which means not to update the docker upstream images, functest, yardstick, etc.' + noapivalidation: + flags: + - '--no-api-validation' + is_flag: 'True' + help: 'disable strict API response validation' diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index 300e8fee..89084e23 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -27,8 +27,7 @@ testsuite_supported: - compliance_set - proposed_tests - debug - - cvp.0.8.0 - - cvp.0.9.0 + - ovp.1.0.0 # testarea supported, should adjust accordingly testarea_supported: - osinterop @@ -87,17 +86,8 @@ validate_input: valid_docker_tag: - 'stable' - 'latest' - - 'danube.1.0' - - 'danube.2.0' - - 'danube.3.0' - - 'danube.3.1' - 'danube.3.2' - - 'cvp.0.1.0' - - 'cvp.0.2.0' - - 'cvp.0.3.0' - - 'cvp.0.4.0' - - 'cvp.0.5.0' - - 'cvp.0.6.0' + - 'ovp.1.0.0' mandatory: - osinterop diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml index 11c49e32..232b0747 100644 --- a/dovetail/conf/functest_config.yml +++ b/dovetail/conf/functest_config.yml @@ -1,7 +1,7 @@ --- functest: image_name: opnfv/functest - docker_tag: cvp.0.5.0 + docker_tag: ovp.1.0.0 opts: '-id --privileged=true' config: dir: '/home/opnfv/userconfig' diff --git a/dovetail/patch/functest/disable-api-validation/0001-Allow-additional-properties-in-API-responses.patch b/dovetail/patch/functest/disable-api-validation/0001-Allow-additional-properties-in-API-responses.patch new file mode 100644 index 00000000..b7a040c4 --- /dev/null +++ b/dovetail/patch/functest/disable-api-validation/0001-Allow-additional-properties-in-API-responses.patch @@ -0,0 +1,1638 @@ +From 9e15ea5e8b15d42eb202363e9a83ae9bb09ccb64 Mon Sep 17 00:00:00 2001 +From: Georg Kunz <georg.kunz@ericsson.com> +Date: Wed, 31 Jan 2018 21:10:35 +0100 +Subject: [PATCH 1/1] Allow additional properties in API responses + +--- + .../lib/api_schema/response/compute/v2_1/agents.py | 10 ++-- + .../api_schema/response/compute/v2_1/aggregates.py | 8 +-- + .../response/compute/v2_1/availability_zone.py | 8 +-- + .../response/compute/v2_1/baremetal_nodes.py | 6 +- + .../response/compute/v2_1/certificates.py | 4 +- + .../api_schema/response/compute/v2_1/extensions.py | 4 +- + .../api_schema/response/compute/v2_1/fixed_ips.py | 4 +- + .../api_schema/response/compute/v2_1/flavors.py | 10 ++-- + .../response/compute/v2_1/flavors_access.py | 4 +- + .../response/compute/v2_1/flavors_extra_specs.py | 2 +- + .../response/compute/v2_1/floating_ips.py | 20 +++---- + .../lib/api_schema/response/compute/v2_1/hosts.py | 14 ++--- + .../response/compute/v2_1/hypervisors.py | 22 ++++---- + .../lib/api_schema/response/compute/v2_1/images.py | 16 +++--- + .../compute/v2_1/instance_usage_audit_logs.py | 8 +-- + .../api_schema/response/compute/v2_1/interfaces.py | 8 +-- + .../api_schema/response/compute/v2_1/keypairs.py | 14 ++--- + .../lib/api_schema/response/compute/v2_1/limits.py | 10 ++-- + .../api_schema/response/compute/v2_1/migrations.py | 4 +- + .../response/compute/v2_1/parameter_types.py | 4 +- + .../lib/api_schema/response/compute/v2_1/quotas.py | 4 +- + .../compute/v2_1/security_group_default_rule.py | 8 +-- + .../response/compute/v2_1/security_groups.py | 16 +++--- + .../api_schema/response/compute/v2_1/servers.py | 64 +++++++++++----------- + .../api_schema/response/compute/v2_1/services.py | 8 +-- + .../api_schema/response/compute/v2_1/snapshots.py | 6 +- + .../response/compute/v2_1/tenant_networks.py | 6 +- + .../api_schema/response/compute/v2_1/versions.py | 10 ++-- + .../api_schema/response/compute/v2_1/volumes.py | 12 ++-- + .../api_schema/response/compute/v2_16/servers.py | 14 ++--- + .../response/compute/v2_23/migrations.py | 4 +- + .../api_schema/response/compute/v2_3/servers.py | 14 ++--- + 32 files changed, 173 insertions(+), 173 deletions(-) + +diff --git a/tempest/lib/api_schema/response/compute/v2_1/agents.py b/tempest/lib/api_schema/response/compute/v2_1/agents.py +index 6f712b4..09feb73 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/agents.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/agents.py +@@ -23,7 +23,7 @@ common_agent_info = { + 'url': {'type': 'string', 'format': 'uri'}, + 'md5hash': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['agent_id', 'hypervisor', 'os', 'architecture', + 'version', 'url', 'md5hash'] + } +@@ -38,7 +38,7 @@ list_agents = { + 'items': common_agent_info + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['agents'] + } + } +@@ -50,7 +50,7 @@ create_agent = { + 'properties': { + 'agent': common_agent_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['agent'] + } + } +@@ -68,11 +68,11 @@ update_agent = { + 'url': {'type': 'string', 'format': 'uri'}, + 'md5hash': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['agent_id', 'version', 'url', 'md5hash'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['agent'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/aggregates.py b/tempest/lib/api_schema/response/compute/v2_1/aggregates.py +index 1a9fe41..4a86670 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/aggregates.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/aggregates.py +@@ -26,7 +26,7 @@ aggregate_for_create = { + 'name': {'type': 'string'}, + 'updated_at': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['availability_zone', 'created_at', 'deleted', + 'deleted_at', 'id', 'name', 'updated_at'], + } +@@ -48,7 +48,7 @@ list_aggregates = { + 'items': common_aggregate_info + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['aggregates'], + } + } +@@ -60,7 +60,7 @@ get_aggregate = { + 'properties': { + 'aggregate': common_aggregate_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['aggregate'], + } + } +@@ -84,7 +84,7 @@ create_aggregate = { + 'properties': { + 'aggregate': aggregate_for_create + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['aggregate'], + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/availability_zone.py b/tempest/lib/api_schema/response/compute/v2_1/availability_zone.py +index d9aebce..7b5e03c 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/availability_zone.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/availability_zone.py +@@ -31,19 +31,19 @@ base = { + 'properties': { + 'available': {'type': 'boolean'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['available'] + }, + # NOTE: Here is the difference between detail and + # non-detail. + 'hosts': {'type': 'null'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['zoneName', 'zoneState', 'hosts'] + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['availabilityZoneInfo'] + } + } +@@ -63,7 +63,7 @@ detail = { + 'active': {'type': 'boolean'}, + 'updated_at': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['available', 'active', 'updated_at'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/baremetal_nodes.py b/tempest/lib/api_schema/response/compute/v2_1/baremetal_nodes.py +index d1ee877..8ab17d3 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/baremetal_nodes.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/baremetal_nodes.py +@@ -25,7 +25,7 @@ node = { + 'memory_mb': {'type': ['integer', 'string']}, + 'disk_gb': {'type': ['integer', 'string']}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'interfaces', 'host', 'task_state', 'cpus', 'memory_mb', + 'disk_gb'] + } +@@ -40,7 +40,7 @@ list_baremetal_nodes = { + 'items': node + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['nodes'] + } + } +@@ -52,7 +52,7 @@ baremetal_node = { + 'properties': { + 'node': node + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['node'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/certificates.py b/tempest/lib/api_schema/response/compute/v2_1/certificates.py +index 4e7cbe4..99f795a 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/certificates.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/certificates.py +@@ -25,11 +25,11 @@ _common_schema = { + 'data': {'type': 'string'}, + 'private_key': {'type': 'string'}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['data', 'private_key'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['certificate'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/extensions.py b/tempest/lib/api_schema/response/compute/v2_1/extensions.py +index a6a455c..9f7395a 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/extensions.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/extensions.py +@@ -35,13 +35,13 @@ list_extensions = { + 'alias': {'type': 'string'}, + 'description': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['updated', 'name', 'links', 'namespace', + 'alias', 'description'] + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['extensions'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/fixed_ips.py b/tempest/lib/api_schema/response/compute/v2_1/fixed_ips.py +index a653213..b53565a 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/fixed_ips.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/fixed_ips.py +@@ -27,11 +27,11 @@ get_fixed_ip = { + 'host': {'type': 'string'}, + 'hostname': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['address', 'cidr', 'host', 'hostname'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['fixed_ip'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/flavors.py b/tempest/lib/api_schema/response/compute/v2_1/flavors.py +index 547d94d..76cbb8a 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/flavors.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/flavors.py +@@ -28,13 +28,13 @@ list_flavors = { + 'links': parameter_types.links, + 'id': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['name', 'links', 'id'] + } + }, + 'flavors_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): flavors_links attribute is not necessary + # to be present always So it is not 'required'. + 'required': ['flavors'] +@@ -58,7 +58,7 @@ common_flavor_info = { + 'rxtx_factor': {'type': 'number'}, + 'OS-FLV-EXT-DATA:ephemeral': {'type': 'integer'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # 'OS-FLV-DISABLED', 'os-flavor-access', 'rxtx_factor' and + # 'OS-FLV-EXT-DATA' are API extensions. So they are not 'required'. + 'required': ['name', 'links', 'ram', 'vcpus', 'swap', 'disk', 'id'] +@@ -77,7 +77,7 @@ list_flavors_details = { + # to be present always So it is not 'required'. + 'flavors_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['flavors'] + } + } +@@ -93,7 +93,7 @@ create_get_flavor_details = { + 'properties': { + 'flavor': common_flavor_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['flavor'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/flavors_access.py b/tempest/lib/api_schema/response/compute/v2_1/flavors_access.py +index a4d6af0..958ed02 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/flavors_access.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/flavors_access.py +@@ -25,12 +25,12 @@ add_remove_list_flavor_access = { + 'flavor_id': {'type': 'string'}, + 'tenant_id': {'type': 'string'}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['flavor_id', 'tenant_id'], + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['flavor_access'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/flavors_extra_specs.py b/tempest/lib/api_schema/response/compute/v2_1/flavors_extra_specs.py +index a438d48..c8988b1 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/flavors_extra_specs.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/flavors_extra_specs.py +@@ -24,7 +24,7 @@ set_get_flavor_extra_specs = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['extra_specs'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/floating_ips.py b/tempest/lib/api_schema/response/compute/v2_1/floating_ips.py +index 0c66590..39e4207 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/floating_ips.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/floating_ips.py +@@ -26,7 +26,7 @@ common_floating_ip_info = { + 'ip': parameter_types.ip_address, + 'fixed_ip': parameter_types.ip_address + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'pool', 'instance_id', + 'ip', 'fixed_ip'], + +@@ -41,7 +41,7 @@ list_floating_ips = { + 'items': common_floating_ip_info + }, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['floating_ips'], + } + } +@@ -53,7 +53,7 @@ create_get_floating_ip = { + 'properties': { + 'floating_ip': common_floating_ip_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['floating_ip'], + } + } +@@ -70,12 +70,12 @@ list_floating_ip_pools = { + 'properties': { + 'name': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['name'], + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['floating_ip_pools'], + } + } +@@ -96,11 +96,11 @@ create_floating_ips_bulk = { + 'ip_range': {'type': 'string'}, + 'pool': {'type': ['string', 'null']}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['interface', 'ip_range', 'pool'], + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['floating_ips_bulk_create'], + } + } +@@ -112,7 +112,7 @@ delete_floating_ips_bulk = { + 'properties': { + 'floating_ips_bulk_delete': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['floating_ips_bulk_delete'], + } + } +@@ -134,7 +134,7 @@ list_floating_ips_bulk = { + 'project_id': {'type': ['string', 'null']}, + 'fixed_ip': parameter_types.ip_address + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE: fixed_ip is introduced after JUNO release, + # So it is not defined as 'required'. + 'required': ['address', 'instance_uuid', 'interface', +@@ -142,7 +142,7 @@ list_floating_ips_bulk = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['floating_ip_info'], + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/hosts.py b/tempest/lib/api_schema/response/compute/v2_1/hosts.py +index ae70ff1..d750cd0 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/hosts.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/hosts.py +@@ -29,12 +29,12 @@ list_hosts = { + 'service': {'type': 'string'}, + 'zone': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['host_name', 'service', 'zone'] + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['hosts'] + } + } +@@ -58,17 +58,17 @@ get_host_detail = { + 'memory_mb': {'type': 'integer'}, + 'project': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['cpu', 'disk_gb', 'host', + 'memory_mb', 'project'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['resource'] + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['host'] + } + } +@@ -81,7 +81,7 @@ startup_host = { + 'host': {'type': 'string'}, + 'power_action': {'enum': ['startup']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['host', 'power_action'] + } + } +@@ -110,7 +110,7 @@ update_host = { + 'off_maintenance']}, + 'status': {'enum': ['enabled', 'disabled']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['host', 'maintenance_mode', 'status'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/hypervisors.py b/tempest/lib/api_schema/response/compute/v2_1/hypervisors.py +index d15b4f6..5d8cf6d 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/hypervisors.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/hypervisors.py +@@ -37,7 +37,7 @@ get_hypervisor_statistics = { + 'vcpus': {'type': 'integer'}, + 'vcpus_used': {'type': 'integer'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['count', 'current_workload', + 'disk_available_least', 'free_disk_gb', + 'free_ram_mb', 'local_gb', 'local_gb_used', +@@ -45,7 +45,7 @@ get_hypervisor_statistics = { + 'vcpus', 'vcpus_used'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['hypervisor_statistics'] + } + } +@@ -78,13 +78,13 @@ hypervisor_detail = { + 'id': {'type': ['integer', 'string']}, + 'disabled_reason': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['host', 'id'] + }, + 'vcpus': {'type': 'integer'}, + 'vcpus_used': {'type': 'integer'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE: When loading os-hypervisor-status extension, + # a response contains status and state. So these params + # should not be required. +@@ -107,7 +107,7 @@ list_hypervisors_detail = { + 'items': hypervisor_detail + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['hypervisors'] + } + } +@@ -119,7 +119,7 @@ get_hypervisor = { + 'properties': { + 'hypervisor': hypervisor_detail + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['hypervisor'] + } + } +@@ -139,7 +139,7 @@ list_search_hypervisors = { + 'id': {'type': ['integer', 'string']}, + 'hypervisor_hostname': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE: When loading os-hypervisor-status extension, + # a response contains status and state. So these params + # should not be required. +@@ -147,7 +147,7 @@ list_search_hypervisors = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['hypervisors'] + } + } +@@ -166,14 +166,14 @@ get_hypervisor_uptime = { + 'hypervisor_hostname': {'type': 'string'}, + 'uptime': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE: When loading os-hypervisor-status extension, + # a response contains status and state. So these params + # should not be required. + 'required': ['id', 'hypervisor_hostname', 'uptime'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['hypervisor'] + } + } +@@ -188,7 +188,7 @@ get_hypervisors_servers['response_body']['properties']['hypervisors']['items'][ + 'uuid': {'type': 'string'}, + 'name': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + } + } + # In V2 API, if there is no servers (VM) on the Hypervisor host then 'servers' +diff --git a/tempest/lib/api_schema/response/compute/v2_1/images.py b/tempest/lib/api_schema/response/compute/v2_1/images.py +index f65b9d8..25d3167 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/images.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/images.py +@@ -40,13 +40,13 @@ common_image_schema = { + 'id': {'type': 'string'}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links'] + }, + 'OS-EXT-IMG-SIZE:size': {'type': ['integer', 'null']}, + 'OS-DCF:diskConfig': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # 'server' attributes only comes in response body if image is + # associated with any server. 'OS-EXT-IMG-SIZE:size' & 'OS-DCF:diskConfig' + # are API extension, So those are not defined as 'required'. +@@ -62,7 +62,7 @@ get_image = { + 'properties': { + 'image': common_image_schema + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['image'] + } + } +@@ -81,13 +81,13 @@ list_images = { + 'links': image_links, + 'name': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links', 'name'] + } + }, + 'images_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): images_links attribute is not necessary to be + # present always So it is not 'required'. + 'required': ['images'] +@@ -120,7 +120,7 @@ image_metadata = { + 'properties': { + 'metadata': {'type': 'object'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['metadata'] + } + } +@@ -132,7 +132,7 @@ image_meta_item = { + 'properties': { + 'meta': {'type': 'object'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['meta'] + } + } +@@ -148,7 +148,7 @@ list_images_details = { + }, + 'images_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): images_links attribute is not necessary to be + # present always So it is not 'required'. + 'required': ['images'] +diff --git a/tempest/lib/api_schema/response/compute/v2_1/instance_usage_audit_logs.py b/tempest/lib/api_schema/response/compute/v2_1/instance_usage_audit_logs.py +index 15224c5..402dfea 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/instance_usage_audit_logs.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/instance_usage_audit_logs.py +@@ -31,7 +31,7 @@ common_instance_usage_audit_log = { + 'errors': {'type': 'integer'}, + 'message': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['state', 'instances', 'errors', 'message'] + } + } +@@ -46,7 +46,7 @@ common_instance_usage_audit_log = { + 'total_errors': {'type': 'integer'}, + 'total_instances': {'type': 'integer'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['hosts_not_run', 'log', 'num_hosts', 'num_hosts_done', + 'num_hosts_not_run', 'num_hosts_running', 'overall_status', + 'period_beginning', 'period_ending', 'total_errors', +@@ -60,7 +60,7 @@ get_instance_usage_audit_log = { + 'properties': { + 'instance_usage_audit_log': common_instance_usage_audit_log + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['instance_usage_audit_log'] + } + } +@@ -72,7 +72,7 @@ list_instance_usage_audit_log = { + 'properties': { + 'instance_usage_audit_logs': common_instance_usage_audit_log + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['instance_usage_audit_logs'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/interfaces.py b/tempest/lib/api_schema/response/compute/v2_1/interfaces.py +index 9984750..6a989e5 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/interfaces.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/interfaces.py +@@ -29,7 +29,7 @@ interface_common_info = { + }, + 'ip_address': parameter_types.ip_address + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['subnet_id', 'ip_address'] + } + }, +@@ -37,7 +37,7 @@ interface_common_info = { + 'net_id': {'type': 'string', 'format': 'uuid'}, + 'mac_addr': parameter_types.mac_address + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['port_state', 'fixed_ips', 'port_id', 'net_id', 'mac_addr'] + } + +@@ -48,7 +48,7 @@ get_create_interfaces = { + 'properties': { + 'interfaceAttachment': interface_common_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['interfaceAttachment'] + } + } +@@ -63,7 +63,7 @@ list_interfaces = { + 'items': interface_common_info + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['interfaceAttachments'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/keypairs.py b/tempest/lib/api_schema/response/compute/v2_1/keypairs.py +index 9c04c79..ec5c2d3 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/keypairs.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/keypairs.py +@@ -31,7 +31,7 @@ get_keypair = { + 'id': {'type': 'integer'} + + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # When we run the get keypair API, response body includes + # all the above mentioned attributes. + # But in Nova API sample file, response body includes only +@@ -40,7 +40,7 @@ get_keypair = { + 'required': ['public_key', 'name', 'fingerprint'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['keypair'] + } + } +@@ -59,14 +59,14 @@ create_keypair = { + 'user_id': {'type': 'string'}, + 'private_key': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # When create keypair API is being called with 'Public key' + # (Importing keypair) then, response body does not contain + # 'private_key' So it is not defined as 'required' + 'required': ['fingerprint', 'name', 'public_key', 'user_id'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['keypair'] + } + } +@@ -92,16 +92,16 @@ list_keypairs = { + 'name': {'type': 'string'}, + 'fingerprint': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['public_key', 'name', 'fingerprint'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['keypair'] + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['keypairs'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/limits.py b/tempest/lib/api_schema/response/compute/v2_1/limits.py +index 81f175f..bc4c1e3 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/limits.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/limits.py +@@ -43,7 +43,7 @@ get_limit = { + 'maxServerGroups': {'type': 'integer'}, + 'totalServerGroupsUsed': {'type': 'integer'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): maxServerGroupMembers, maxServerGroups + # and totalServerGroupsUsed are API extension, + # and some environments return a response without these +@@ -86,21 +86,21 @@ get_limit = { + 'verb': + {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + } + }, + 'regex': {'type': 'string'}, + 'uri': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['absolute', 'rate'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['limits'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/migrations.py b/tempest/lib/api_schema/response/compute/v2_1/migrations.py +index b7d66ea..b571820 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/migrations.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/migrations.py +@@ -35,7 +35,7 @@ list_migrations = { + 'created_at': {'type': 'string'}, + 'updated_at': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': [ + 'id', 'status', 'instance_uuid', 'source_node', + 'source_compute', 'dest_node', 'dest_compute', +@@ -45,7 +45,7 @@ list_migrations = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['migrations'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/parameter_types.py b/tempest/lib/api_schema/response/compute/v2_1/parameter_types.py +index 3cc5ca4..73843d1 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/parameter_types.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/parameter_types.py +@@ -23,7 +23,7 @@ links = { + }, + 'rel': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['href', 'rel'] + } + } +@@ -74,7 +74,7 @@ addresses = { + ] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['version', 'addr'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/quotas.py b/tempest/lib/api_schema/response/compute/v2_1/quotas.py +index 7953983..f4d9153 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/quotas.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/quotas.py +@@ -37,7 +37,7 @@ update_quota_set = { + 'injected_file_content_bytes': {'type': 'integer'}, + 'injected_file_path_bytes': {'type': 'integer'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE: server_group_members and server_groups are represented + # when enabling quota_server_group extension. So they should + # not be required. +@@ -49,7 +49,7 @@ update_quota_set = { + 'injected_file_path_bytes'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['quota_set'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/security_group_default_rule.py b/tempest/lib/api_schema/response/compute/v2_1/security_group_default_rule.py +index 2ec2826..1a2e19b 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/security_group_default_rule.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/security_group_default_rule.py +@@ -23,12 +23,12 @@ common_security_group_default_rule_info = { + 'properties': { + 'cidr': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['cidr'], + }, + 'to_port': {'type': 'integer'}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['from_port', 'id', 'ip_protocol', 'ip_range', 'to_port'], + } + +@@ -40,7 +40,7 @@ create_get_security_group_default_rule = { + 'security_group_default_rule': + common_security_group_default_rule_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['security_group_default_rule'] + } + } +@@ -59,7 +59,7 @@ list_security_group_default_rules = { + 'items': common_security_group_default_rule_info + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['security_group_default_rules'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/security_groups.py b/tempest/lib/api_schema/response/compute/v2_1/security_groups.py +index 5ed5a5c..d9f1794 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/security_groups.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/security_groups.py +@@ -21,7 +21,7 @@ common_security_group_rule = { + 'tenant_id': {'type': 'string'}, + 'name': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + }, + 'ip_protocol': {'type': ['string', 'null']}, + # 'parent_group_id' can be UUID so defining it as 'string' also. +@@ -31,7 +31,7 @@ common_security_group_rule = { + 'properties': { + 'cidr': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # When optional argument is provided in request body + # like 'group_id' then, attribute 'cidr' does not + # comes in response body. So it is not 'required'. +@@ -50,12 +50,12 @@ common_security_group = { + 'items': { + 'type': ['object', 'null'], + 'properties': common_security_group_rule, +- 'additionalProperties': False, ++ 'additionalProperties': True, + } + }, + 'description': {'type': 'string'}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'name', 'tenant_id', 'rules', 'description'], + } + +@@ -69,7 +69,7 @@ list_security_groups = { + 'items': common_security_group + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['security_groups'] + } + } +@@ -81,7 +81,7 @@ get_security_group = create_security_group = update_security_group = { + 'properties': { + 'security_group': common_security_group + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['security_group'] + } + } +@@ -98,12 +98,12 @@ create_security_group_rule = { + 'security_group_rule': { + 'type': 'object', + 'properties': common_security_group_rule, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['from_port', 'to_port', 'group', 'ip_protocol', + 'parent_group_id', 'id', 'ip_range'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['security_group_rule'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/servers.py b/tempest/lib/api_schema/response/compute/v2_1/servers.py +index 63e8467..8f4b385 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/servers.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/servers.py +@@ -29,14 +29,14 @@ create_server = { + 'links': parameter_types.links, + 'OS-DCF:diskConfig': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE: OS-DCF:diskConfig & security_groups are API extension, + # and some environments return a response without these + # attributes.So they are not 'required'. + 'required': ['id', 'links'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['server'] + } + } +@@ -61,13 +61,13 @@ list_servers = { + 'links': parameter_types.links, + 'name': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links', 'name'] + } + }, + 'servers_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): servers_links attribute is not necessary to be + # present always So it is not 'required'. + 'required': ['servers'] +@@ -90,7 +90,7 @@ common_show_server = { + 'id': {'type': 'string'}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links']}, + {'type': ['string', 'null']} + ]}, +@@ -100,7 +100,7 @@ common_show_server = { + 'id': {'type': 'string'}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links'] + }, + 'fault': { +@@ -111,7 +111,7 @@ common_show_server = { + 'message': {'type': 'string'}, + 'details': {'type': 'string'}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): 'details' is not necessary to be present + # in the 'fault'. So it is not defined as 'required'. + 'required': ['code', 'created', 'message'] +@@ -129,7 +129,7 @@ common_show_server = { + 'accessIPv4': parameter_types.access_ip_v4, + 'accessIPv6': parameter_types.access_ip_v6 + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(GMann): 'progress' attribute is present in the response + # only when server's status is one of the progress statuses + # ("ACTIVE","BUILD", "REBUILD", "RESIZE","VERIFY_RESIZE") +@@ -150,7 +150,7 @@ update_server = { + 'properties': { + 'server': common_show_server + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['server'] + } + } +@@ -181,7 +181,7 @@ server_detail['properties'].update({ + 'properties': { + 'id': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + }, + }, + 'config_drive': {'type': 'string'} +@@ -202,7 +202,7 @@ get_server = { + 'properties': { + 'server': server_detail + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['server'] + } + } +@@ -218,7 +218,7 @@ list_servers_detail = { + }, + 'servers_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): servers_links attribute is not necessary to be + # present always So it is not 'required'. + 'required': ['servers'] +@@ -241,7 +241,7 @@ rescue_server = { + 'properties': { + 'adminPass': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['adminPass'] + } + } +@@ -260,14 +260,14 @@ list_virtual_interfaces = { + 'mac_address': parameter_types.mac_address, + 'OS-EXT-VIF-NET:net_id': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # 'OS-EXT-VIF-NET:net_id' is API extension So it is + # not defined as 'required' + 'required': ['id', 'mac_address'] + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['virtual_interfaces'] + } + } +@@ -280,7 +280,7 @@ common_attach_volume_info = { + 'volumeId': {'type': 'string'}, + 'serverId': {'type': ['integer', 'string']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # 'device' is optional in response. + 'required': ['id', 'volumeId', 'serverId'] + } +@@ -292,7 +292,7 @@ attach_volume = { + 'properties': { + 'volumeAttachment': common_attach_volume_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['volumeAttachment'] + } + } +@@ -315,7 +315,7 @@ list_volume_attachments = { + 'items': common_attach_volume_info + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['volumeAttachments'] + } + } +@@ -335,7 +335,7 @@ list_addresses = { + 'properties': { + 'addresses': parameter_types.addresses + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['addresses'] + } + } +@@ -357,7 +357,7 @@ common_server_group = { + }, + 'metadata': {'type': 'object'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'name', 'policies', 'members', 'metadata'] + } + +@@ -368,7 +368,7 @@ create_show_server_group = { + 'properties': { + 'server_group': common_server_group + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['server_group'] + } + } +@@ -387,7 +387,7 @@ list_server_groups = { + 'items': common_server_group + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['server_groups'] + } + } +@@ -403,7 +403,7 @@ instance_actions = { + 'message': {'type': ['string', 'null']}, + 'instance_uuid': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['action', 'request_id', 'user_id', 'project_id', + 'start_time', 'message', 'instance_uuid'] + } +@@ -419,7 +419,7 @@ instance_action_events = { + 'result': {'type': 'string'}, + 'traceback': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['event', 'start_time', 'finish_time', 'result', + 'traceback'] + } +@@ -435,7 +435,7 @@ list_instance_actions = { + 'items': instance_actions + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['instanceActions'] + } + } +@@ -453,7 +453,7 @@ show_instance_action = { + 'properties': { + 'instanceAction': instance_actions_with_events + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['instanceAction'] + } + } +@@ -465,7 +465,7 @@ show_password = { + 'properties': { + 'password': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['password'] + } + } +@@ -484,11 +484,11 @@ get_vnc_console = { + 'format': 'uri' + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['type', 'url'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['console'] + } + } +@@ -500,7 +500,7 @@ get_console_output = { + 'properties': { + 'output': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['output'] + } + } +@@ -517,7 +517,7 @@ set_server_metadata = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['metadata'] + } + } +@@ -542,7 +542,7 @@ set_show_server_metadata_item = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['meta'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/services.py b/tempest/lib/api_schema/response/compute/v2_1/services.py +index ddef7b2..4b490d1 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/services.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/services.py +@@ -32,13 +32,13 @@ list_services = { + 'updated_at': {'type': ['string', 'null']}, + 'disabled_reason': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'zone', 'host', 'state', 'binary', + 'status', 'updated_at', 'disabled_reason'] + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['services'] + } + } +@@ -55,11 +55,11 @@ enable_disable_service = { + 'binary': {'type': 'string'}, + 'host': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['status', 'binary', 'host'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['service'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/snapshots.py b/tempest/lib/api_schema/response/compute/v2_1/snapshots.py +index 01a524b..4638dd2 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/snapshots.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/snapshots.py +@@ -24,7 +24,7 @@ common_snapshot_info = { + 'displayName': {'type': ['string', 'null']}, + 'displayDescription': {'type': ['string', 'null']} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'volumeId', 'status', 'size', + 'createdAt', 'displayName', 'displayDescription'] + } +@@ -36,7 +36,7 @@ create_get_snapshot = { + 'properties': { + 'snapshot': common_snapshot_info + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['snapshot'] + } + } +@@ -51,7 +51,7 @@ list_snapshots = { + 'items': common_snapshot_info + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['snapshots'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/tenant_networks.py b/tempest/lib/api_schema/response/compute/v2_1/tenant_networks.py +index ddfab96..02a9382 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/tenant_networks.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/tenant_networks.py +@@ -19,7 +19,7 @@ param_network = { + 'cidr': {'type': ['string', 'null']}, + 'label': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'cidr', 'label'] + } + +@@ -34,7 +34,7 @@ list_tenant_networks = { + 'items': param_network + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['networks'] + } + } +@@ -47,7 +47,7 @@ get_tenant_network = { + 'properties': { + 'network': param_network + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['network'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/versions.py b/tempest/lib/api_schema/response/compute/v2_1/versions.py +index 08a9fab..d6c1021 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/versions.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/versions.py +@@ -29,7 +29,7 @@ _version = { + 'type': {'type': 'string'}, + }, + 'required': ['href', 'rel'], +- 'additionalProperties': False ++ 'additionalProperties': True + } + }, + 'status': {'type': 'string'}, +@@ -48,7 +48,7 @@ _version = { + # so they should not be required. + # NOTE(sdague): media-types only shows up in single version requests. + 'required': ['id', 'links', 'status', 'updated'], +- 'additionalProperties': False ++ 'additionalProperties': True + } + + list_versions = { +@@ -62,7 +62,7 @@ list_versions = { + } + }, + 'required': ['versions'], +- 'additionalProperties': False ++ 'additionalProperties': True + } + } + +@@ -94,7 +94,7 @@ get_version = { + } + }, + 'required': ['choices'], +- 'additionalProperties': False ++ 'additionalProperties': True + } + } + +@@ -105,6 +105,6 @@ get_one_version = { + 'properties': { + 'version': _version + }, +- 'additionalProperties': False ++ 'additionalProperties': True + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_1/volumes.py b/tempest/lib/api_schema/response/compute/v2_1/volumes.py +index bb34acb..d854d53 100644 +--- a/tempest/lib/api_schema/response/compute/v2_1/volumes.py ++++ b/tempest/lib/api_schema/response/compute/v2_1/volumes.py +@@ -40,7 +40,7 @@ create_get_volume = { + 'volumeId': {'type': 'string'}, + 'serverId': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE- If volume is not attached to any server + # then, 'attachments' attributes comes as array + # with empty objects "[{}]" due to that elements +@@ -50,13 +50,13 @@ create_get_volume = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'status', 'displayName', 'availabilityZone', + 'createdAt', 'displayDescription', 'volumeType', + 'snapshotId', 'metadata', 'size', 'attachments'] + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['volume'] + } + } +@@ -91,7 +91,7 @@ list_volumes = { + 'volumeId': {'type': 'string'}, + 'serverId': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE- If volume is not attached to any server + # then, 'attachments' attributes comes as array + # with empty object "[{}]" due to that elements +@@ -101,7 +101,7 @@ list_volumes = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'status', 'displayName', + 'availabilityZone', 'createdAt', + 'displayDescription', 'volumeType', +@@ -110,7 +110,7 @@ list_volumes = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['volumes'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_16/servers.py b/tempest/lib/api_schema/response/compute/v2_16/servers.py +index 3eb658f..d0a30e3 100644 +--- a/tempest/lib/api_schema/response/compute/v2_16/servers.py ++++ b/tempest/lib/api_schema/response/compute/v2_16/servers.py +@@ -32,7 +32,7 @@ server_detail = { + 'id': {'type': 'string'}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links']}, + {'type': ['string', 'null']} + ]}, +@@ -42,7 +42,7 @@ server_detail = { + 'id': {'type': 'string'}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links'] + }, + 'fault': { +@@ -53,7 +53,7 @@ server_detail = { + 'message': {'type': 'string'}, + 'details': {'type': 'string'}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): 'details' is not necessary to be present + # in the 'fault'. So it is not defined as 'required'. + 'required': ['code', 'created', 'message'] +@@ -90,7 +90,7 @@ server_detail = { + 'id': {'type': 'string'}, + 'delete_on_termination': {'type': 'boolean'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + }, + }, + 'OS-EXT-SRV-ATTR:reservation_id': {'type': ['string', 'null']}, +@@ -104,7 +104,7 @@ server_detail = { + # NOTE(gmann): new attributes in version 2.16 + 'host_status': {'type': 'string'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): 'progress' attribute is present in the response + # only when server's status is one of the progress statuses + # ("ACTIVE","BUILD", "REBUILD", "RESIZE","VERIFY_RESIZE") +@@ -134,7 +134,7 @@ get_server = { + 'properties': { + 'server': server_detail + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['server'] + } + } +@@ -150,7 +150,7 @@ list_servers_detail = { + }, + 'servers_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): servers_links attribute is not necessary to be + # present always So it is not 'required'. + 'required': ['servers'] +diff --git a/tempest/lib/api_schema/response/compute/v2_23/migrations.py b/tempest/lib/api_schema/response/compute/v2_23/migrations.py +index 3cd0f6e..af6fd8a 100644 +--- a/tempest/lib/api_schema/response/compute/v2_23/migrations.py ++++ b/tempest/lib/api_schema/response/compute/v2_23/migrations.py +@@ -45,7 +45,7 @@ list_migrations = { + 'migration_type': {'type': ['string', 'null']}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': [ + 'id', 'status', 'instance_uuid', 'source_node', + 'source_compute', 'dest_node', 'dest_compute', +@@ -56,7 +56,7 @@ list_migrations = { + } + } + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['migrations'] + } + } +diff --git a/tempest/lib/api_schema/response/compute/v2_3/servers.py b/tempest/lib/api_schema/response/compute/v2_3/servers.py +index f24103e..5b5c9c1 100644 +--- a/tempest/lib/api_schema/response/compute/v2_3/servers.py ++++ b/tempest/lib/api_schema/response/compute/v2_3/servers.py +@@ -40,7 +40,7 @@ server_detail = { + 'id': {'type': 'string'}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links']}, + {'type': ['string', 'null']} + ]}, +@@ -50,7 +50,7 @@ server_detail = { + 'id': {'type': 'string'}, + 'links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['id', 'links'] + }, + 'fault': { +@@ -61,7 +61,7 @@ server_detail = { + 'message': {'type': 'string'}, + 'details': {'type': 'string'}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): 'details' is not necessary to be present + # in the 'fault'. So it is not defined as 'required'. + 'required': ['code', 'created', 'message'] +@@ -99,7 +99,7 @@ server_detail = { + 'id': {'type': 'string'}, + 'delete_on_termination': {'type': 'boolean'} + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + }, + }, + 'OS-EXT-SRV-ATTR:reservation_id': {'type': ['string', 'null']}, +@@ -110,7 +110,7 @@ server_detail = { + 'OS-EXT-SRV-ATTR:root_device_name': {'type': ['string', 'null']}, + 'OS-EXT-SRV-ATTR:user_data': {'type': ['string', 'null']}, + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): 'progress' attribute is present in the response + # only when server's status is one of the progress statuses + # ("ACTIVE","BUILD", "REBUILD", "RESIZE","VERIFY_RESIZE") +@@ -140,7 +140,7 @@ get_server = { + 'properties': { + 'server': server_detail + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + 'required': ['server'] + } + } +@@ -156,7 +156,7 @@ list_servers_detail = { + }, + 'servers_links': parameter_types.links + }, +- 'additionalProperties': False, ++ 'additionalProperties': True, + # NOTE(gmann): servers_links attribute is not necessary to be + # present always So it is not 'required'. + 'required': ['servers'] +-- +2.7.4 + diff --git a/dovetail/patch/functest/disable-api-validation/apply.sh b/dovetail/patch/functest/disable-api-validation/apply.sh new file mode 100755 index 00000000..915bce43 --- /dev/null +++ b/dovetail/patch/functest/disable-api-validation/apply.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +set -u + +# without setting the user, git does not allow to create a commit +git config --global user.email "verified@opnfv.org" +git config --global user.name "Dovetail" + +cd /src/tempest +git am $(dirname $0)/0001-Allow-additional-properties-in-API-responses.patch + +exit 0 diff --git a/dovetail/run.py b/dovetail/run.py index 4778fa51..5ff22323 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -105,7 +105,7 @@ def validate_input(input_dict, check_dict, logger): # for 'func_tag' and 'yard_tag' options func_tag = input_dict['func_tag'] yard_tag = input_dict['yard_tag'] - bott_tag = input_dict['bott_tag'] + # bott_tag = input_dict['bott_tag'] valid_tag = check_dict['valid_docker_tag'] if func_tag is not None and func_tag not in valid_tag: logger.error("The input option 'func_tag' can't be {}, " @@ -115,10 +115,10 @@ def validate_input(input_dict, check_dict, logger): logger.error("The input option 'yard_tag' can't be {}, " "valid values are {}.".format(yard_tag, valid_tag)) raise SystemExit(1) - if bott_tag is not None and bott_tag not in valid_tag: - logger.error("The input option 'bott_tag' can't be {}, " - "valid values are {}.".format(bott_tag, valid_tag)) - raise SystemExit(1) + # if bott_tag is not None and bott_tag not in valid_tag: + # logger.error("The input option 'bott_tag' can't be {}, " + # "valid values are {}.".format(bott_tag, valid_tag)) + # raise SystemExit(1) # for 'report' option report = input_dict['report'] @@ -221,7 +221,7 @@ def copy_patch_files(logger): patch_set_path = dt_cfg.dovetail_config['patch_dir'] if not os.path.isdir(patch_set_path): os.makedirs(patch_set_path) - cmd = 'sudo cp -r %s/* %s' % (patch_path, patch_set_path) + cmd = 'sudo cp -a -r %s/* %s' % (patch_path, patch_set_path) dt_utils.exec_cmd(cmd, logger, exit_on_error=False) @@ -281,6 +281,12 @@ def main(*args, **kwargs): else: dt_cfg.dovetail_config['offline'] = False + if kwargs['no_api_validation']: + dt_cfg.dovetail_config['no_api_validation'] = True + logger.warning('Strict API response validation DISABLED.') + else: + dt_cfg.dovetail_config['no_api_validation'] = False + dt_utils.get_hardware_info(logger) origin_testarea = kwargs['testarea'] diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 99845484..05c63eb7 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -290,6 +290,25 @@ class FunctestTestcase(Testcase): super(FunctestTestcase, self).__init__(testcase_yaml) self.type = 'functest' + def prepare_cmd(self, test_type): + if not super(FunctestTestcase, self).prepare_cmd(test_type): + return False + + # if API validation is disabled, append a command for applying a + # patch inside the functest container + if dt_cfg.dovetail_config['no_api_validation']: + patch_cmd = os.path.join( + dt_cfg.dovetail_config['functest']['config']['dir'], + 'patch', + 'functest', + 'disable-api-validation', + 'apply.sh') + self.cmds = [patch_cmd] + self.cmds + self.logger.debug('Updated list of commands for test run with ' + 'disabled API response validation: {}' + .format(self.cmds)) + return True + class YardstickTestcase(Testcase): diff --git a/dovetail/utils/local_db/launch_db.sh b/dovetail/utils/local_db/launch_db.sh index cad0f365..7d80cc7a 100755 --- a/dovetail/utils/local_db/launch_db.sh +++ b/dovetail/utils/local_db/launch_db.sh @@ -52,7 +52,7 @@ echo "Create the testapi service." echo "==========================" set +e -testapi_img="opnfv/testapi:cvp.0.3.0" +testapi_img="opnfv/testapi:ovp.1.0.0" echo "Step1: pull the image $testapi_img." sudo docker pull $testapi_img set -e @@ -1,6 +1,6 @@ [metadata] name = dovetail -version = 0.9.0 +version = 1.0.0 home-page = https://wiki.opnfv.org/display/dovetail [entry_points] |
