aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xci/exec_test.sh3
-rwxr-xr-xci/run_tests.py1
-rw-r--r--ci/testcases.yaml11
-rw-r--r--docker/Dockerfile2
-rw-r--r--docker/requirements.pip3
-rwxr-xr-xtestcases/OpenStack/tempest/run_tempest.py59
-rwxr-xr-xtestcases/features/copper.py13
-rwxr-xr-xutils/openstack_clean.py24
8 files changed, 82 insertions, 34 deletions
diff --git a/ci/exec_test.sh b/ci/exec_test.sh
index 7e57cd5e4..26067a5d3 100755
--- a/ci/exec_test.sh
+++ b/ci/exec_test.sh
@@ -166,6 +166,9 @@ function run_test(){
"domino")
python ${FUNCTEST_REPO_DIR}/testcases/features/domino.py
;;
+ "odl-sfc")
+ python ${FUNCTEST_REPO_DIR}/testcases/features/sfc/sfc.py
+ ;;
*)
echo "The test case '${test_name}' does not exist."
exit 1
diff --git a/ci/run_tests.py b/ci/run_tests.py
index f172eecc6..a024dd720 100755
--- a/ci/run_tests.py
+++ b/ci/run_tests.py
@@ -104,7 +104,6 @@ def run_test(test, tier_name):
duration_str = ("%02d:%02d" % divmod(duration, 60))
logger.info("Test execution time: %s" % duration_str)
- result = 0
if result != 0:
logger.error("The test case '%s' failed. " % test_name)
OVERALL_RESULT = -1
diff --git a/ci/testcases.yaml b/ci/testcases.yaml
index 84dd8e483..e5182c19a 100644
--- a/ci/testcases.yaml
+++ b/ci/testcases.yaml
@@ -151,7 +151,7 @@ tiers:
Simple security Scan
dependencies:
installer: 'apex'
- scenario: ''
+ scenario: 'disable'
-
name: copper
@@ -189,6 +189,15 @@ tiers:
dependencies:
installer: 'joid'
scenario: ''
+ -
+ name: odl-sfc
+ criteria: 'status == "PASS"'
+ blocking: false
+ description: >-
+ Test suite for odl-sfc to test two chains and two SFs
+ dependencies:
+ installer: 'fuel'
+ scenario: 'odl_l2-sfc'
-
name: openstack
order: 4
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 3053382da..e138c73dc 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -70,7 +70,7 @@ RUN git config --global http.sslVerify false
RUN git clone https://gerrit.opnfv.org/gerrit/functest ${repos_dir}/functest
RUN git clone https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng
RUN git clone https://gerrit.opnfv.org/gerrit/doctor ${repos_dir}/doctor
-RUN git clone -b 0.3.3 https://github.com/openstack/rally.git ${repos_dir}/rally
+RUN git clone -b 0.5.0 https://github.com/openstack/rally.git ${repos_dir}/rally
RUN git clone https://github.com/openstack/tempest.git ${repos_dir}/tempest
RUN git clone https://git.opendaylight.org/gerrit/p/integration/test.git ${repos_dir}/odl_test
RUN git clone -b stable https://github.com/boucherv-orange/clearwater-live-test ${repos_dir}/vims-test
diff --git a/docker/requirements.pip b/docker/requirements.pip
index fa6681e8e..f590b2a03 100644
--- a/docker/requirements.pip
+++ b/docker/requirements.pip
@@ -23,4 +23,5 @@ xmltodict==0.9.2
scp==0.10.2
paramiko==1.16.0
python-congressclient==1.3.0
-subprocess32 \ No newline at end of file
+subprocess32
+shyaml
diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py
index 5d1648dc0..8ca3bdb3f 100755
--- a/testcases/OpenStack/tempest/run_tempest.py
+++ b/testcases/OpenStack/tempest/run_tempest.py
@@ -219,7 +219,7 @@ def configure_tempest_feature(deployment_dir, mode):
if mode == 'feature_multisite':
config.set('service_available', 'kingbird', 'true')
cmd = "openstack endpoint show kingbird | grep publicurl |\
- awk '{print $4}' | awk -F '/' '{print $3}'"
+ awk '{print $4}' | awk -F '/' '{print $4}'"
kingbird_api_version = os.popen(cmd).read()
if os.environ.get("INSTALLER_TYPE") == 'fuel':
# For MOS based setup, the service is accessible
@@ -397,23 +397,41 @@ def run_tempest(OPTION):
dur_sec_int = int(round(dur_sec_float, 0))
dur_sec_int = dur_sec_int + 60 * dur_min
stop_time = time.time()
- # Push results in payload of testcase
- if args.report:
- logger.debug("Pushing tempest results into DB...")
- # Note criteria hardcoded...TODO move to testcase.yaml
- status = "FAIL"
- try:
- diff = (int(num_tests) - int(num_failures))
- success_rate = 100 * diff / int(num_tests)
- except:
- success_rate = 0
- # For Tempest we assume that the success rate is above 90%
- if success_rate >= 90:
+ status = "FAIL"
+ try:
+ diff = (int(num_tests) - int(num_failures))
+ success_rate = 100 * diff / int(num_tests)
+ except:
+ success_rate = 0
+
+ # For Tempest we assume that the success rate is above 90%
+ if "smoke" in args.mode:
+ case_name = "tempest_smoke_serial"
+ # Note criteria hardcoded...TODO read it from testcases.yaml
+ success_criteria = 100
+ if success_rate >= success_criteria:
+ status = "PASS"
+ else:
+ logger.info("Tempest success rate: %s%%. The success criteria to "
+ "pass this test is %s%%. Marking the test as FAILED." %
+ (success_rate, success_criteria))
+ else:
+ case_name = "tempest_full_parallel"
+ # Note criteria hardcoded...TODO read it from testcases.yaml
+ success_criteria = 80
+ if success_rate >= success_criteria:
status = "PASS"
+ else:
+ logger.info("Tempest success rate: %s%%. The success criteria to "
+ "pass this test is %s%%. Marking the test as FAILED." %
+ (success_rate, success_criteria))
+ # Push results in payload of testcase
+ if args.report:
# add the test in error in the details sections
# should be possible to do it during the test
+ logger.debug("Pushing tempest results into DB...")
with open(TEMPEST_RESULTS_DIR + "/tempest.log", 'r') as myfile:
output = myfile.read()
error_logs = ""
@@ -427,10 +445,6 @@ def run_tempest(OPTION):
"errors": error_logs}
logger.info("Results: " + str(json_results))
# split Tempest smoke and full
- if "smoke" in args.mode:
- case_name = "tempest_smoke_serial"
- else:
- case_name = "tempest_full_parallel"
try:
ft_utils.push_results_to_db("functest",
@@ -444,6 +458,11 @@ def run_tempest(OPTION):
logger.error("Error pushing results into Database '%s'"
% sys.exc_info()[0])
+ if status == "PASS":
+ return 0
+ else:
+ return -1
+
def main():
global MODE
@@ -467,7 +486,11 @@ def main():
if args.serial:
MODE += " --concur 1"
- run_tempest(MODE)
+ ret_val = run_tempest(MODE)
+ if ret_val != 0:
+ sys.exit(-1)
+
+ sys.exit(0)
if __name__ == '__main__':
diff --git a/testcases/features/copper.py b/testcases/features/copper.py
index 7ab4e78aa..9efcbd7f4 100755
--- a/testcases/features/copper.py
+++ b/testcases/features/copper.py
@@ -16,11 +16,12 @@
#
import os
+import sys
import time
-import yaml
-
import functest.utils.functest_logger as ft_logger
import functest.utils.functest_utils as functest_utils
+import yaml
+
with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
functest_yaml = yaml.safe_load(f)
@@ -38,11 +39,11 @@ def main():
start_time = time.time()
- ret = functest_utils.execute_command(cmd, logger, exit_on_error=False)
+ ret_val = functest_utils.execute_command(cmd, logger, exit_on_error=False)
stop_time = time.time()
duration = round(stop_time - start_time, 1)
- if ret == 0:
+ if ret_val == 0:
logger.info("COPPER PASSED")
test_status = 'PASS'
else:
@@ -76,6 +77,10 @@ def main():
stop_time,
details['status'],
details)
+ if ret_val != 0:
+ sys.exit(-1)
+
+ sys.exit(0)
if __name__ == '__main__':
main()
diff --git a/utils/openstack_clean.py b/utils/openstack_clean.py
index ce82bc154..8aba763ce 100755
--- a/utils/openstack_clean.py
+++ b/utils/openstack_clean.py
@@ -48,7 +48,8 @@ def remove_instances(nova_client, default_instances):
instance_name = getattr(instance, 'name')
instance_id = getattr(instance, 'id')
logger.debug("'%s', ID=%s " % (instance_name, instance_id))
- if instance_id not in default_instances:
+ if (instance_id not in default_instances and
+ instance_name not in default_instances.values()):
logger.debug("Removing instance '%s' ..." % instance_id)
if os_utils.delete_instance(nova_client, instance_id):
logger.debug(" > Request sent.")
@@ -83,7 +84,8 @@ def remove_images(nova_client, default_images):
image_name = getattr(image, 'name')
image_id = getattr(image, 'id')
logger.debug("'%s', ID=%s " % (image_name, image_id))
- if image_id not in default_images:
+ if (image_id not in default_images and
+ image_name not in default_images.values()):
logger.debug("Removing image '%s', ID=%s ..."
% (image_name, image_id))
if os_utils.delete_glance_image(nova_client, image_id):
@@ -107,7 +109,8 @@ def remove_volumes(cinder_client, default_volumes):
volume_id = getattr(volume, 'id')
volume_name = getattr(volume, 'display_name')
logger.debug("'%s', ID=%s " % (volume_name, volume_id))
- if volume_id not in default_volumes:
+ if (volume_id not in default_volumes and
+ volume_name not in default_volumes.values()):
logger.debug("Removing cinder volume %s ..." % volume_id)
if os_utils.delete_volume(cinder_client, volume_id):
logger.debug(" > Done!")
@@ -138,7 +141,8 @@ def remove_floatingips(nova_client, default_floatingips):
fip_id = getattr(fip, 'id')
fip_ip = getattr(fip, 'ip')
logger.debug("'%s', ID=%s " % (fip_ip, fip_id))
- if fip_id not in default_floatingips:
+ if (fip_id not in default_floatingips and
+ fip_ip not in default_floatingips.values()):
logger.debug("Removing floating IP %s ..." % fip_id)
if os_utils.delete_floating_ip(nova_client, fip_id):
logger.debug(" > Done!")
@@ -173,7 +177,8 @@ def remove_networks(neutron_client, default_networks, default_routers):
net_id = network['id']
net_name = network['name']
logger.debug(" '%s', ID=%s " % (net_name, net_id))
- if net_id in default_networks:
+ if (net_id in default_networks and
+ net_name in default_networks.values()):
logger.debug(" > this is a default network and will "
"NOT be deleted.")
elif network['router:external'] is True:
@@ -260,7 +265,8 @@ def remove_routers(neutron_client, routers, default_routers):
for router in routers:
router_id = router['id']
router_name = router['name']
- if router_id not in default_routers:
+ if (router_id not in default_routers and
+ router_name not in default_routers.values()):
logger.debug("Checking '%s' with ID=(%s) ..." % (router_name,
router_id))
if router['external_gateway_info'] is not None:
@@ -317,7 +323,8 @@ def remove_users(keystone_client, default_users):
user_name = getattr(user, 'name')
user_id = getattr(user, 'id')
logger.debug("'%s', ID=%s " % (user_name, user_id))
- if user_id not in default_users:
+ if (user_id not in default_users and
+ user_name not in default_users.values()):
logger.debug(" Removing '%s'..." % user_name)
if os_utils.delete_user(keystone_client, user_id):
logger.debug(" > Done!")
@@ -340,7 +347,8 @@ def remove_tenants(keystone_client, default_tenants):
tenant_name = getattr(tenant, 'name')
tenant_id = getattr(tenant, 'id')
logger.debug("'%s', ID=%s " % (tenant_name, tenant_id))
- if tenant_id not in default_tenants:
+ if (tenant_id not in default_tenants and
+ tenant_name not in default_tenants.values()):
logger.debug(" Removing '%s'..." % tenant_name)
if os_utils.delete_tenant(keystone_client, tenant_id):
logger.debug(" > Done!")