diff options
| -rwxr-xr-x | ci/exec_test.sh | 3 | ||||
| -rwxr-xr-x | ci/run_tests.py | 1 | ||||
| -rw-r--r-- | ci/testcases.yaml | 11 | ||||
| -rw-r--r-- | docker/Dockerfile | 2 | ||||
| -rw-r--r-- | docker/requirements.pip | 3 | ||||
| -rwxr-xr-x | testcases/OpenStack/tempest/run_tempest.py | 59 | ||||
| -rwxr-xr-x | testcases/features/copper.py | 13 | ||||
| -rwxr-xr-x | utils/openstack_clean.py | 24 |
8 files changed, 82 insertions, 34 deletions
diff --git a/ci/exec_test.sh b/ci/exec_test.sh index 7e57cd5e4..26067a5d3 100755 --- a/ci/exec_test.sh +++ b/ci/exec_test.sh @@ -166,6 +166,9 @@ function run_test(){ "domino") python ${FUNCTEST_REPO_DIR}/testcases/features/domino.py ;; + "odl-sfc") + python ${FUNCTEST_REPO_DIR}/testcases/features/sfc/sfc.py + ;; *) echo "The test case '${test_name}' does not exist." exit 1 diff --git a/ci/run_tests.py b/ci/run_tests.py index f172eecc6..a024dd720 100755 --- a/ci/run_tests.py +++ b/ci/run_tests.py @@ -104,7 +104,6 @@ def run_test(test, tier_name): duration_str = ("%02d:%02d" % divmod(duration, 60)) logger.info("Test execution time: %s" % duration_str) - result = 0 if result != 0: logger.error("The test case '%s' failed. " % test_name) OVERALL_RESULT = -1 diff --git a/ci/testcases.yaml b/ci/testcases.yaml index 84dd8e483..e5182c19a 100644 --- a/ci/testcases.yaml +++ b/ci/testcases.yaml @@ -151,7 +151,7 @@ tiers: Simple security Scan dependencies: installer: 'apex' - scenario: '' + scenario: 'disable' - name: copper @@ -189,6 +189,15 @@ tiers: dependencies: installer: 'joid' scenario: '' + - + name: odl-sfc + criteria: 'status == "PASS"' + blocking: false + description: >- + Test suite for odl-sfc to test two chains and two SFs + dependencies: + installer: 'fuel' + scenario: 'odl_l2-sfc' - name: openstack order: 4 diff --git a/docker/Dockerfile b/docker/Dockerfile index 3053382da..e138c73dc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -70,7 +70,7 @@ RUN git config --global http.sslVerify false RUN git clone https://gerrit.opnfv.org/gerrit/functest ${repos_dir}/functest RUN git clone https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng RUN git clone https://gerrit.opnfv.org/gerrit/doctor ${repos_dir}/doctor -RUN git clone -b 0.3.3 https://github.com/openstack/rally.git ${repos_dir}/rally +RUN git clone -b 0.5.0 https://github.com/openstack/rally.git ${repos_dir}/rally RUN git clone https://github.com/openstack/tempest.git ${repos_dir}/tempest RUN git clone https://git.opendaylight.org/gerrit/p/integration/test.git ${repos_dir}/odl_test RUN git clone -b stable https://github.com/boucherv-orange/clearwater-live-test ${repos_dir}/vims-test diff --git a/docker/requirements.pip b/docker/requirements.pip index fa6681e8e..f590b2a03 100644 --- a/docker/requirements.pip +++ b/docker/requirements.pip @@ -23,4 +23,5 @@ xmltodict==0.9.2 scp==0.10.2 paramiko==1.16.0 python-congressclient==1.3.0 -subprocess32
\ No newline at end of file +subprocess32 +shyaml diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py index 5d1648dc0..8ca3bdb3f 100755 --- a/testcases/OpenStack/tempest/run_tempest.py +++ b/testcases/OpenStack/tempest/run_tempest.py @@ -219,7 +219,7 @@ def configure_tempest_feature(deployment_dir, mode): if mode == 'feature_multisite': config.set('service_available', 'kingbird', 'true') cmd = "openstack endpoint show kingbird | grep publicurl |\ - awk '{print $4}' | awk -F '/' '{print $3}'" + awk '{print $4}' | awk -F '/' '{print $4}'" kingbird_api_version = os.popen(cmd).read() if os.environ.get("INSTALLER_TYPE") == 'fuel': # For MOS based setup, the service is accessible @@ -397,23 +397,41 @@ def run_tempest(OPTION): dur_sec_int = int(round(dur_sec_float, 0)) dur_sec_int = dur_sec_int + 60 * dur_min stop_time = time.time() - # Push results in payload of testcase - if args.report: - logger.debug("Pushing tempest results into DB...") - # Note criteria hardcoded...TODO move to testcase.yaml - status = "FAIL" - try: - diff = (int(num_tests) - int(num_failures)) - success_rate = 100 * diff / int(num_tests) - except: - success_rate = 0 - # For Tempest we assume that the success rate is above 90% - if success_rate >= 90: + status = "FAIL" + try: + diff = (int(num_tests) - int(num_failures)) + success_rate = 100 * diff / int(num_tests) + except: + success_rate = 0 + + # For Tempest we assume that the success rate is above 90% + if "smoke" in args.mode: + case_name = "tempest_smoke_serial" + # Note criteria hardcoded...TODO read it from testcases.yaml + success_criteria = 100 + if success_rate >= success_criteria: + status = "PASS" + else: + logger.info("Tempest success rate: %s%%. The success criteria to " + "pass this test is %s%%. Marking the test as FAILED." % + (success_rate, success_criteria)) + else: + case_name = "tempest_full_parallel" + # Note criteria hardcoded...TODO read it from testcases.yaml + success_criteria = 80 + if success_rate >= success_criteria: status = "PASS" + else: + logger.info("Tempest success rate: %s%%. The success criteria to " + "pass this test is %s%%. Marking the test as FAILED." % + (success_rate, success_criteria)) + # Push results in payload of testcase + if args.report: # add the test in error in the details sections # should be possible to do it during the test + logger.debug("Pushing tempest results into DB...") with open(TEMPEST_RESULTS_DIR + "/tempest.log", 'r') as myfile: output = myfile.read() error_logs = "" @@ -427,10 +445,6 @@ def run_tempest(OPTION): "errors": error_logs} logger.info("Results: " + str(json_results)) # split Tempest smoke and full - if "smoke" in args.mode: - case_name = "tempest_smoke_serial" - else: - case_name = "tempest_full_parallel" try: ft_utils.push_results_to_db("functest", @@ -444,6 +458,11 @@ def run_tempest(OPTION): logger.error("Error pushing results into Database '%s'" % sys.exc_info()[0]) + if status == "PASS": + return 0 + else: + return -1 + def main(): global MODE @@ -467,7 +486,11 @@ def main(): if args.serial: MODE += " --concur 1" - run_tempest(MODE) + ret_val = run_tempest(MODE) + if ret_val != 0: + sys.exit(-1) + + sys.exit(0) if __name__ == '__main__': diff --git a/testcases/features/copper.py b/testcases/features/copper.py index 7ab4e78aa..9efcbd7f4 100755 --- a/testcases/features/copper.py +++ b/testcases/features/copper.py @@ -16,11 +16,12 @@ #
import os
+import sys
import time
-import yaml
-
import functest.utils.functest_logger as ft_logger
import functest.utils.functest_utils as functest_utils
+import yaml
+
with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
functest_yaml = yaml.safe_load(f)
@@ -38,11 +39,11 @@ def main(): start_time = time.time()
- ret = functest_utils.execute_command(cmd, logger, exit_on_error=False)
+ ret_val = functest_utils.execute_command(cmd, logger, exit_on_error=False)
stop_time = time.time()
duration = round(stop_time - start_time, 1)
- if ret == 0:
+ if ret_val == 0:
logger.info("COPPER PASSED")
test_status = 'PASS'
else:
@@ -76,6 +77,10 @@ def main(): stop_time,
details['status'],
details)
+ if ret_val != 0:
+ sys.exit(-1)
+
+ sys.exit(0)
if __name__ == '__main__':
main()
diff --git a/utils/openstack_clean.py b/utils/openstack_clean.py index ce82bc154..8aba763ce 100755 --- a/utils/openstack_clean.py +++ b/utils/openstack_clean.py @@ -48,7 +48,8 @@ def remove_instances(nova_client, default_instances): instance_name = getattr(instance, 'name') instance_id = getattr(instance, 'id') logger.debug("'%s', ID=%s " % (instance_name, instance_id)) - if instance_id not in default_instances: + if (instance_id not in default_instances and + instance_name not in default_instances.values()): logger.debug("Removing instance '%s' ..." % instance_id) if os_utils.delete_instance(nova_client, instance_id): logger.debug(" > Request sent.") @@ -83,7 +84,8 @@ def remove_images(nova_client, default_images): image_name = getattr(image, 'name') image_id = getattr(image, 'id') logger.debug("'%s', ID=%s " % (image_name, image_id)) - if image_id not in default_images: + if (image_id not in default_images and + image_name not in default_images.values()): logger.debug("Removing image '%s', ID=%s ..." % (image_name, image_id)) if os_utils.delete_glance_image(nova_client, image_id): @@ -107,7 +109,8 @@ def remove_volumes(cinder_client, default_volumes): volume_id = getattr(volume, 'id') volume_name = getattr(volume, 'display_name') logger.debug("'%s', ID=%s " % (volume_name, volume_id)) - if volume_id not in default_volumes: + if (volume_id not in default_volumes and + volume_name not in default_volumes.values()): logger.debug("Removing cinder volume %s ..." % volume_id) if os_utils.delete_volume(cinder_client, volume_id): logger.debug(" > Done!") @@ -138,7 +141,8 @@ def remove_floatingips(nova_client, default_floatingips): fip_id = getattr(fip, 'id') fip_ip = getattr(fip, 'ip') logger.debug("'%s', ID=%s " % (fip_ip, fip_id)) - if fip_id not in default_floatingips: + if (fip_id not in default_floatingips and + fip_ip not in default_floatingips.values()): logger.debug("Removing floating IP %s ..." % fip_id) if os_utils.delete_floating_ip(nova_client, fip_id): logger.debug(" > Done!") @@ -173,7 +177,8 @@ def remove_networks(neutron_client, default_networks, default_routers): net_id = network['id'] net_name = network['name'] logger.debug(" '%s', ID=%s " % (net_name, net_id)) - if net_id in default_networks: + if (net_id in default_networks and + net_name in default_networks.values()): logger.debug(" > this is a default network and will " "NOT be deleted.") elif network['router:external'] is True: @@ -260,7 +265,8 @@ def remove_routers(neutron_client, routers, default_routers): for router in routers: router_id = router['id'] router_name = router['name'] - if router_id not in default_routers: + if (router_id not in default_routers and + router_name not in default_routers.values()): logger.debug("Checking '%s' with ID=(%s) ..." % (router_name, router_id)) if router['external_gateway_info'] is not None: @@ -317,7 +323,8 @@ def remove_users(keystone_client, default_users): user_name = getattr(user, 'name') user_id = getattr(user, 'id') logger.debug("'%s', ID=%s " % (user_name, user_id)) - if user_id not in default_users: + if (user_id not in default_users and + user_name not in default_users.values()): logger.debug(" Removing '%s'..." % user_name) if os_utils.delete_user(keystone_client, user_id): logger.debug(" > Done!") @@ -340,7 +347,8 @@ def remove_tenants(keystone_client, default_tenants): tenant_name = getattr(tenant, 'name') tenant_id = getattr(tenant, 'id') logger.debug("'%s', ID=%s " % (tenant_name, tenant_id)) - if tenant_id not in default_tenants: + if (tenant_id not in default_tenants and + tenant_name not in default_tenants.values()): logger.debug(" Removing '%s'..." % tenant_name) if os_utils.delete_tenant(keystone_client, tenant_id): logger.debug(" > Done!") |