diff options
| -rwxr-xr-x | ci/exec_test.sh | 2 | ||||
| -rw-r--r-- | docs/com/img/OPNFV_testing_group.png | bin | 0 -> 87295 bytes | |||
| -rw-r--r-- | docs/com/img/teamSummitBerlin.png | bin | 845680 -> 899923 bytes | |||
| -rwxr-xr-x | docs/com/pres/conversation.html | 61 | ||||
| -rwxr-xr-x | docs/com/pres/testapi.html | 26 | ||||
| -rw-r--r-- | testcases/Controllers/ODL/custom_tests/neutron/040__delete_ports.robot | 14 | ||||
| -rw-r--r-- | testcases/Controllers/ODL/custom_tests/neutron/050__delete_subnets.robot | 14 | ||||
| -rw-r--r-- | testcases/Controllers/ODL/custom_tests/neutron/060__delete_networks.robot | 16 | ||||
| -rw-r--r-- | testcases/Controllers/ODL/odlreport2db.py | 4 | ||||
| -rw-r--r-- | testcases/Controllers/ONOS/Teston/onosfunctest.py | 2 | ||||
| -rwxr-xr-x | testcases/OpenStack/rally/run_rally-cert.py | 9 | ||||
| -rw-r--r-- | testcases/OpenStack/tempest/run_tempest.py | 11 | ||||
| -rw-r--r-- | testcases/OpenStack/vPing/vPing_ssh.py | 2 | ||||
| -rw-r--r-- | testcases/OpenStack/vPing/vPing_userdata.py | 2 | ||||
| -rw-r--r-- | testcases/security_scan/security_scan.py | 7 | ||||
| -rw-r--r-- | testcases/vIMS/vIMS.py | 4 | ||||
| -rw-r--r-- | utils/openstack_utils.py | 17 |
17 files changed, 134 insertions, 57 deletions
diff --git a/ci/exec_test.sh b/ci/exec_test.sh index 04f38e6a8..9d8d83d66 100755 --- a/ci/exec_test.sh +++ b/ci/exec_test.sh @@ -134,7 +134,7 @@ function run_test(){ # ${repos_dir}/ovno/Testcases/RunTests.sh ;; "security_scan") - echo "Sourcing Credentials ${FUNCTEST_CONF_DIR}/stackrc for security_scan test.." + echo "Sourcing Credentials ${FUNCTEST_CONF_DIR}/stackrc for undercloud .." source ${FUNCTEST_CONF_DIR}/stackrc python ${FUNCTEST_REPO_DIR}/testcases/security_scan/security_scan.py --config ${FUNCTEST_REPO_DIR}/testcases/security_scan/config.ini ;; diff --git a/docs/com/img/OPNFV_testing_group.png b/docs/com/img/OPNFV_testing_group.png Binary files differnew file mode 100644 index 000000000..4b8c2c053 --- /dev/null +++ b/docs/com/img/OPNFV_testing_group.png diff --git a/docs/com/img/teamSummitBerlin.png b/docs/com/img/teamSummitBerlin.png Binary files differindex f4fc8cb4f..d5c7f46aa 100644 --- a/docs/com/img/teamSummitBerlin.png +++ b/docs/com/img/teamSummitBerlin.png diff --git a/docs/com/pres/conversation.html b/docs/com/pres/conversation.html index 0fab448a2..df14f5142 100755 --- a/docs/com/pres/conversation.html +++ b/docs/com/pres/conversation.html @@ -63,7 +63,8 @@ <section data-markdown> # Who are we? -  +  + This is a representation...not a governance! </section> <section> <section> @@ -83,7 +84,7 @@ <tr> <td>Yardstick</td> <td>Umbrella project for performance/qualification testing</td> - </tr> + </tr> <tr> <td>CPerf</td> <td>SDN controller performance testing</td> @@ -98,7 +99,11 @@ </tr> <tr> <td>Bottlenecks</td> - <td>Detect possible bottlenecks </td> + <td>Detect possible bottlenecks</td> + </tr> + <tr> + <td>QTIP</td> + <td>Benchmark to boost performance</td> </tr> </tbody> </table> @@ -118,10 +123,10 @@ * Performance testing framework and NFVI Validation * [Contributors](https://wiki.opnfv.org/display/yardstick/People) * Yardstick presentation - * How to Do a Pre-deployment NFVI Validation Quickly and Efficiently? Wednesday, June 22, 4:00 PM - 4:30 PM - * Yardstick: A Pre-deployment NFVI Validation Tool. Thursday, June 23, 3:10 PM - 3:30 PM + * How to Do a Pre-deployment NFVI Validation Quickly and Efficiently? Wednesday, June 22, 4:00 PM - 4:30 PM + * Yardstick: A Pre-deployment NFVI Validation Tool. Thursday, June 23, 3:10 PM - 3:30 PM * breakout sessions - * Tomorrow 2PM (Yardstick in C) + * Tomorrow 2PM (Yardstick in C) </section> <section data-markdown> # CPerf @@ -131,6 +136,13 @@ </section> <section data-markdown> # StorPerf + * Cinder Volume Performance Testing + * Based on SNIA's SSD Test Specification + * Presentation + * StorPerf: Cinder Storage Performance Measurement. + Wednesday, June 22, 3:40 PM - 3:55 PM + * Breakout Session + * Tomorrow 10:15 (StorPerf Test Results Working Session) </section> <section data-markdown> # VSPerf @@ -138,21 +150,38 @@ <section data-markdown> # Bottlenecks </section> + <section data-markdown> + # QTIP + + ## OPNFV Platform Benchmark + + - Bottom up + - testing baremetal components first + - More than result + - comparing against reference + - Setup-Test-Diagnose-Improve + - a closed loop to boost platform performance + + ## Breakout session + + - 14:30~15:45, June 21st/Tuesday + - Room Tegel + </section> </section> <section> <section data-markdown> - # Upstream, upstream, upstream.. + # Upstream, upstream, upstream.. </section> <section data-markdown> ## We are on the shoulders of the giants * Rally (OpenStack) * ODL, ONOS, ... * RobotFramework, Teston - * ..... + * ..... </section> <section data-markdown> - ## Time to give back - ### How to improve work with testing upstream community + ## Time to give back + ### How to improve work with testing upstream community </section> </section> @@ -162,20 +191,20 @@ <section data-markdown> # Towards Telco Cloud KPI? </section> - + <section data-markdown> # Testing group goals * Define common needs (naming, rules, tools...) - * Unify result collection, reporting, dashboarding, documentation - * Share best practice: docker, CLI, .. + * Unify result collection, reporting, dashboarding, documentation + * Share best practice: docker, CLI, .. * Identify testing domains - * Encourage testing in poorly covered areas + * Encourage testing in poorly covered areas </section> <section data-markdown> # The ultimate goal: define Telco Cloud KPIs * Provide methodology and test suites to qualify a Telco Cloud - * Provide reference benchs for plugfest/third party integration: compare apple with apple + * Provide reference benchs for plugfest/third party integration: compare apple with apple </section> </section> @@ -186,7 +215,7 @@ </div> <div class='footer'> - <img src="../img/logo-OPNFV.png" alt="OPNFV logo"> + <img src="../img/logo-OPNFV-Berlin.png" alt="OPNFV logo"> </div> </div> diff --git a/docs/com/pres/testapi.html b/docs/com/pres/testapi.html index 37a8ea356..a0e266a19 100755 --- a/docs/com/pres/testapi.html +++ b/docs/com/pres/testapi.html @@ -62,9 +62,9 @@ </section> <section data-markdown> ## Give a consistant view for + * Test resources (Pods) * Test projects * Test cases - * Test resources (Pods) * Test results </section> @@ -87,17 +87,9 @@ # API overview </section> <section data-markdown> - ## API in Brahmaputra - * Tornado + MongoDB - * Simple data models aggreed with the testing group - * No unit tests - * Wiki and rst documentation - </section> - - <section data-markdown> - ## API in Brahmaputra + ## API storage structure  - </section> + </section> <section data-markdown> ## API in Brahmaputra  @@ -105,6 +97,14 @@ </section> <section data-markdown> + ## API in Brahmaputra + * Tornado + MongoDB + * Simple data models aggreed with the testing group + * No unit tests + * Wiki and rst documentation + </section> + + <section data-markdown> ## Lessons learned in B. * Wiki documentation is painful * Result model too simple @@ -137,8 +137,8 @@ testresults.opnfv.org/test/testapi => testresults.opnfv.org/test/api/v1 http://testresults.opnfv.org/test/api/v1/projects - http://testresults.opnfv.org/test/api/v1/projects/yardstick/cases - http://testresults.opnfv.org/test/api/v1/results?case=Tempest&period=1 + http://testresults.opnfv.org/test/api/v1/projects/qtip/cases + http://testresults.opnfv.org/test/api/v1/results?case=Tempest&period=1 </section> <section> <h3>One new field for pods</h3> diff --git a/testcases/Controllers/ODL/custom_tests/neutron/040__delete_ports.robot b/testcases/Controllers/ODL/custom_tests/neutron/040__delete_ports.robot index 7260d78de..b934ca5d3 100644 --- a/testcases/Controllers/ODL/custom_tests/neutron/040__delete_ports.robot +++ b/testcases/Controllers/ODL/custom_tests/neutron/040__delete_ports.robot @@ -1,6 +1,6 @@ *** Settings *** Documentation Checking Port deleted in OpenStack are deleted also in OpenDaylight -Suite Setup Create Session OSSession http://${NEUTRON}:9696 headers=${X-AUTH} +Suite Setup Start Suite Suite Teardown Delete All Sessions Library RequestsLibrary Variables ../../../variables/Variables.py @@ -23,7 +23,6 @@ Delete New Port Check Port Deleted [Documentation] Check port deleted in OpenDaylight [Tags] Check port deleted OpenDaylight - Create Session ODLSession http://${CONTROLLER}:${PORT} headers=${HEADERS} auth=${AUTH} ${resp} get request ODLSession ${ODLREST} Should be Equal As Strings ${resp.status_code} 200 ${ODLResult} To Json ${resp.content} @@ -31,3 +30,14 @@ Check Port Deleted Log ${ODLResult} ${resp} get request ODLSession ${ODLREST}/${PORTID} Should be Equal As Strings ${resp.status_code} 404 + +*** Keywords *** +Check Port Exists + [Arguments] ${portid} + ${resp} get request ODLSession ${ODLREST}/${portid} + Should be Equal As Strings ${resp.status_code} 200 + +Start Suite + Create Session OSSession http://${NEUTRON}:9696 headers=${X-AUTH} + Create Session ODLSession http://${CONTROLLER}:${PORT} headers=${HEADERS} auth=${AUTH} + Check Port Exists ${PORTID} diff --git a/testcases/Controllers/ODL/custom_tests/neutron/050__delete_subnets.robot b/testcases/Controllers/ODL/custom_tests/neutron/050__delete_subnets.robot index ede9123f4..16e396fd0 100644 --- a/testcases/Controllers/ODL/custom_tests/neutron/050__delete_subnets.robot +++ b/testcases/Controllers/ODL/custom_tests/neutron/050__delete_subnets.robot @@ -1,6 +1,6 @@ *** Settings *** Documentation Checking Subnets deleted in OpenStack are deleted also in OpenDaylight -Suite Setup Create Session OSSession http://${NEUTRON}:9696 headers=${X-AUTH} +Suite Setup Start Suite Suite Teardown Delete All Sessions Library RequestsLibrary Variables ../../../variables/Variables.py @@ -23,7 +23,6 @@ Delete New subnet Check New subnet deleted [Documentation] Check subnet deleted in OpenDaylight [Tags] Check subnet deleted OpenDaylight - Create Session ODLSession http://${CONTROLLER}:${PORT} headers=${HEADERS} auth=${AUTH} ${resp} get request ODLSession ${ODLREST} Should be Equal As Strings ${resp.status_code} 200 ${ODLResult} To Json ${resp.content} @@ -31,3 +30,14 @@ Check New subnet deleted Log ${ODLResult} ${resp} get request ODLSession ${ODLREST}/${SUBNETID} Should be Equal As Strings ${resp.status_code} 404 + +*** Keywords *** +Check Subnet Exists + [Arguments] ${subnetid} + ${resp} get request ODLSession ${ODLREST}/${subnetid} + Should be Equal As Strings ${resp.status_code} 200 + +Start Suite + Create Session OSSession http://${NEUTRON}:9696 headers=${X-AUTH} + Create Session ODLSession http://${CONTROLLER}:${PORT} headers=${HEADERS} auth=${AUTH} + Check Subnet Exists ${SUBNETID} diff --git a/testcases/Controllers/ODL/custom_tests/neutron/060__delete_networks.robot b/testcases/Controllers/ODL/custom_tests/neutron/060__delete_networks.robot index d917632b5..574e13248 100644 --- a/testcases/Controllers/ODL/custom_tests/neutron/060__delete_networks.robot +++ b/testcases/Controllers/ODL/custom_tests/neutron/060__delete_networks.robot @@ -1,6 +1,6 @@ *** Settings *** Documentation Checking Network deleted in OpenStack are deleted also in OpenDaylight -Suite Setup Create Session OSSession http://${NEUTRON}:9696 headers=${X-AUTH} +Suite Setup Start Suite Suite Teardown Delete All Sessions Library RequestsLibrary Variables ../../../variables/Variables.py @@ -23,11 +23,21 @@ Delete Network Check Network deleted [Documentation] Check network deleted in OpenDaylight [Tags] Check Network OpenDaylight - Create Session ODLSession http://${CONTROLLER}:${PORT} headers=${HEADERS} auth=${AUTH} ${resp} get request ODLSession ${ODLREST} Should be Equal As Strings ${resp.status_code} 200 ${ODLResult} To Json ${resp.content} Set Suite Variable ${ODLResult} Log ${ODLResult} - ${resp} get request ODLSession ${ODLREST}/${NetID} + ${resp} get request ODLSession ${ODLREST}/${NETID} Should be Equal As Strings ${resp.status_code} 404 + +*** Keywords *** +Check Network Exists + [Arguments] ${netid} + ${resp} get request ODLSession ${ODLREST}/${netid} + Should be Equal As Strings ${resp.status_code} 200 + +Start Suite + Create Session OSSession http://${NEUTRON}:9696 headers=${X-AUTH} + Create Session ODLSession http://${CONTROLLER}:${PORT} headers=${HEADERS} auth=${AUTH} + Check Network Exists ${NETID} diff --git a/testcases/Controllers/ODL/odlreport2db.py b/testcases/Controllers/ODL/odlreport2db.py index 075de5a86..0c9f341c9 100644 --- a/testcases/Controllers/ODL/odlreport2db.py +++ b/testcases/Controllers/ODL/odlreport2db.py @@ -115,7 +115,7 @@ def main(argv): data['description'] = all_data['suite']['@name'] data['version'] = all_data['@generator'] data['test_project'] = "functest" - data['case_name'] = "ODL" + data['case_name'] = "odl" data['pod_name'] = pod data['installer'] = installer @@ -144,7 +144,7 @@ def main(argv): if (tests_failed < 1): status = "PASS" - functest_utils.push_results_to_db("functest", + functest_utils.push_results_to_db(data['test_project'], data['case_name'], None, start_time, diff --git a/testcases/Controllers/ONOS/Teston/onosfunctest.py b/testcases/Controllers/ONOS/Teston/onosfunctest.py index 38935c5dd..058114bde 100644 --- a/testcases/Controllers/ONOS/Teston/onosfunctest.py +++ b/testcases/Controllers/ONOS/Teston/onosfunctest.py @@ -192,7 +192,7 @@ def main(): logger.error("Unable to set ONOS criteria") functest_utils.push_results_to_db("functest", - "ONOS", + "onos", logger, start_time, stop_time, diff --git a/testcases/OpenStack/rally/run_rally-cert.py b/testcases/OpenStack/rally/run_rally-cert.py index 7822c7e9e..8afbccaa3 100755 --- a/testcases/OpenStack/rally/run_rally-cert.py +++ b/testcases/OpenStack/rally/run_rally-cert.py @@ -535,11 +535,16 @@ def main(): if total_success >= 90: status = "passed" + if args.sanity: + case_name = "rally_sanity" + else: + case_name = "rally_full" + if args.report: logger.debug("Pushing Rally summary into DB...") functest_utils.push_results_to_db("functest", - "Rally", - logger, + case_name, + None, start_time, stop_time, status, diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py index f59ba5263..a16ba26f7 100644 --- a/testcases/OpenStack/tempest/run_tempest.py +++ b/testcases/OpenStack/tempest/run_tempest.py @@ -333,11 +333,16 @@ def run_tempest(OPTION): "tests": int(num_tests), "failures": int(num_failures), "errors": error_logs} logger.info("Results: " + str(json_results)) - # TODO split Tempest smoke and full + # split Tempest smoke and full + if "smoke" in args.mode: + case_name = "tempest_smoke_serial" + else: + case_name = "tempest_full_parallel" + try: ft_utils.push_results_to_db("functest", - "Tempest", - logger, + case_name, + None, start_time, stop_time, status, diff --git a/testcases/OpenStack/vPing/vPing_ssh.py b/testcases/OpenStack/vPing/vPing_ssh.py index 3f90bc214..1609dd95b 100644 --- a/testcases/OpenStack/vPing/vPing_ssh.py +++ b/testcases/OpenStack/vPing/vPing_ssh.py @@ -429,7 +429,7 @@ def main(): try: logger.debug("Pushing vPing SSH results into DB...") functest_utils.push_results_to_db("functest", - "vPing", + "vping_ssh", logger, start_time, stop_time, diff --git a/testcases/OpenStack/vPing/vPing_userdata.py b/testcases/OpenStack/vPing/vPing_userdata.py index 5b7d2d90f..ff7158ee3 100644 --- a/testcases/OpenStack/vPing/vPing_userdata.py +++ b/testcases/OpenStack/vPing/vPing_userdata.py @@ -362,7 +362,7 @@ def main(): try: logger.debug("Pushing vPing userdata results into DB...") functest_utils.push_results_to_db("functest", - "vPing_userdata", + "vping_userdata", logger, start_time, stop_time, diff --git a/testcases/security_scan/security_scan.py b/testcases/security_scan/security_scan.py index 7ed85b64d..5910331a0 100644 --- a/testcases/security_scan/security_scan.py +++ b/testcases/security_scan/security_scan.py @@ -16,6 +16,7 @@ import argparse import connect import datetime import os +import sys from ConfigParser import SafeConfigParser from keystoneclient.auth.identity import v2 @@ -27,8 +28,8 @@ __author__ = 'Luke Hinds (lhinds@redhat.com)' __url__ = 'https://wiki.opnfv.org/display/functest/Functest+Security' # Global vars -oscapbin = 'sudo /bin/oscap' INSTALLER_IP = os.getenv('INSTALLER_IP') +oscapbin = 'sudo /bin/oscap' # Apex Spefic var needed to query Undercloud if os.getenv('OS_AUTH_URL') is None: @@ -37,7 +38,7 @@ if os.getenv('OS_AUTH_URL') is None: else: OS_AUTH_URL = os.getenv('OS_AUTH_URL') -# argparse +# args parser = argparse.ArgumentParser(description='OPNFV OpenSCAP Scanner') parser.add_argument('--config', action='store', dest='cfgfile', help='Config file', required=True) @@ -58,7 +59,7 @@ setup.getockey() com = 'sudo hiera admin_password' setup = connect.SetUp(com) keypass = setup.keystonepass() -auth = v2.Password(auth_url='http://{0}:5000/v2.0'.format(OS_AUTH_URL), +auth = v2.Password(auth_url=OS_AUTH_URL, username='admin', password=str(keypass).rstrip(), tenant_name='admin') diff --git a/testcases/vIMS/vIMS.py b/testcases/vIMS/vIMS.py index 260cb4ce2..a35eba98d 100644 --- a/testcases/vIMS/vIMS.py +++ b/testcases/vIMS/vIMS.py @@ -122,8 +122,8 @@ def step_failure(step_name, error_msg): if step_name == "sig_test": status = "PASS" functest_utils.push_results_to_db("functest", - "vIMS", - logger, + "vims", + None, start_time, stop_time, status, diff --git a/utils/openstack_utils.py b/utils/openstack_utils.py index 4084e1fc2..646df7aec 100644 --- a/utils/openstack_utils.py +++ b/utils/openstack_utils.py @@ -41,6 +41,14 @@ def get_credentials(service): requesting the credentials. """ creds = {} + + # Check that the env vars exists: + envvars = ('OS_USERNAME', 'OS_PASSWORD', 'OS_AUTH_URL', 'OS_TENANT_NAME') + for envvar in envvars: + if os.getenv(envvar) is None: + print("'%s' is not exported as an env variable." % envvar) + exit(-1) + # Unfortunately, each of the OpenStack client will request slightly # different entries in their credentials dict. if service.lower() in ("nova", "cinder"): @@ -53,11 +61,10 @@ def get_credentials(service): # The most common way to pass these info to the script is to do it through # environment variables. creds.update({ - "username": os.environ.get('OS_USERNAME', "admin"), - password: os.environ.get("OS_PASSWORD", 'admin'), - "auth_url": os.environ.get("OS_AUTH_URL", - "http://192.168.20.71:5000/v2.0"), - tenant: os.environ.get("OS_TENANT_NAME", "admin"), + "username": os.environ.get("OS_USERNAME"), + password: os.environ.get("OS_PASSWORD"), + "auth_url": os.environ.get("OS_AUTH_URL"), + tenant: os.environ.get("OS_TENANT_NAME") }) cacert = os.environ.get("OS_CACERT") if cacert is not None: |
