diff options
| -rw-r--r-- | docs/com/pres/gates/gates.md | 82 | ||||
| -rw-r--r-- | docs/com/pres/gates/index.html | 52 | ||||
| -rw-r--r-- | docs/testing/user/userguide/test_details.rst | 28 | ||||
| -rw-r--r-- | docs/testing/user/userguide/test_results.rst | 36 | ||||
| -rw-r--r-- | functest/api/resources/v1/testcases.py | 4 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/rally.py | 2 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/scenario/full/opnfv-gnocchi.yaml | 163 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-gnocchi.yaml | 110 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/task.yaml | 4 | ||||
| -rw-r--r-- | functest/opnfv_tests/vnf/epc/juju_epc.py | 6 | ||||
| -rw-r--r-- | functest/opnfv_tests/vnf/ims/cloudify_ims_perf.yaml | 4 |
11 files changed, 484 insertions, 7 deletions
diff --git a/docs/com/pres/gates/gates.md b/docs/com/pres/gates/gates.md new file mode 100644 index 000000000..aaf087743 --- /dev/null +++ b/docs/com/pres/gates/gates.md @@ -0,0 +1,82 @@ +# Functest Gates + +[Cédric Ollivier](mailto:cedric.ollivier@orange.com) + +2018/04/19 + + + +### Why gating? + +- maintain an overall high quality code whatever the skills involved in our +open community +- detect the errors as soon as possible (before merge operations) +- verify automatically most of Functest milestones + + +### Today's Verify -1 + +- one error is detected by yamllint, pylint or doc8 +- one unit test fails (py27 and py35) +- our Framework is not fully covered +- specific modules are not rated 10/10 by pylint +- unix permissions are wrong +- one doc cannot be generated without warning + +Please see [tox.ini](https://git.opnfv.org/functest/tree/tox.ini) for details + + +### Gambia's Verify -1 + +- enforce an overall 10/10 by pylint +- check the full coverage of all third-party interfaces +- integrate other linters like ShellScript (if they meet tox basics) + +Please see +[Gambia Challenges](http://testresults.opnfv.org/functest/gambiachallenges/) +for details + + +### Great! But + +- the current gate checks all coding rules, our Framework and all interfaces +to third-parties but **not our testcases** +- it doesn't break **the circular dependencies** between Installers and +Functest (Installers ask for a trustable healthcheck but we need results of +"Installer runs" as prerequisites) + +Our testcases have still **to be verified by hand** what differs from +OpenStack Workflow + + + +### functional gating + +- we need to run all Functest patches before merge against reference +plateforms (OpenStack and Kubernetes) +- a voting job must forbid the merge if one test fails (as the workflow +process implemented by OpenStack) + + +### reference platform + +- it could be baremetal or virtual if all testcases (including all VNFs) can +be tested successfully +- any compliant Installer could be selected as long as the appropriate target +VIM version is proposed at the beginning of the release: + - devstack (OpenStack gates) + - kolla-ansible (Orange ONAP OpenLab) + - XCI + - APEX + + +### Verify +2 + +Only the functional check will take hours once the patchset is accepted +by core reviewers (each patchset will be verified in ~10 minutes). + +**Always stable!** + + + +## Thank you! diff --git a/docs/com/pres/gates/index.html b/docs/com/pres/gates/index.html new file mode 100644 index 000000000..b64e371f7 --- /dev/null +++ b/docs/com/pres/gates/index.html @@ -0,0 +1,52 @@ +<html> +<head> +<title>Functest Gates</title> +<meta name="author" content="Cédric Ollivier"> +<meta name="viewport" + content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"> +<link rel="stylesheet" href="../reveal.js/css/reveal.css"> +<link rel="stylesheet" href="../reveal.js/css/theme/white.css"> +<link rel="stylesheet" href="../reveal.js/lib/css/zenburn.css"> +<script> +var link = document.createElement( 'link' ); +link.rel = 'stylesheet'; +link.type = 'text/css'; +link.href = window.location.search.match( /print-pdf/gi ) ? '../reveal.js/css/print/pdf.css' : '../reveal.js/css/print/paper.css'; +document.getElementsByTagName( 'head' )[0].appendChild( link ); +</script> +</head> +<body> + <div class="reveal"> + <div class="slides"> + <section data-markdown="gates.md" data-separator="^\n\n\n" + data-separator-vertical="^\n\n" data-separator-notes="^Note:"></section> + </div> + </div> + <script src="../reveal.js/lib/js/head.min.js"></script> + <script src="../reveal.js/js/reveal.js"></script> + <script> + Reveal.initialize({ + dependencies : [ { + src : '../reveal.js/plugin/markdown/marked.js', + condition : function() { + return !!document.querySelector('[data-markdown]'); + } + }, { + src : '../reveal.js/plugin/markdown/markdown.js', + condition : function() { + return !!document.querySelector('[data-markdown]'); + } + }, { + src: '../reveal.js/plugin/highlight/highlight.js', + async: true, + callback: function() { + hljs.initHighlightingOnLoad(); + } + }, { + src: '../reveal.js/plugin/notes/notes.js', + async: true + } ] + }); + </script> +</body> +</html> diff --git a/docs/testing/user/userguide/test_details.rst b/docs/testing/user/userguide/test_details.rst index b7b9d8846..002b3b7e6 100644 --- a/docs/testing/user/userguide/test_details.rst +++ b/docs/testing/user/userguide/test_details.rst @@ -423,6 +423,33 @@ The vyos-vrouter architecture is described in `[14]`_ juju_epc ^^^^^^^^ + +Kubernetes (K8s) +---------------- + +Kubernetes testing relies on sets of tests, which are part of the Kubernetes +source tree, such as the Kubernetes End-to-End (e2e) tests `[15]`_. + +The kubernetes testcases are distributed across various Tiers: + + * Healthcheck Tier + + * k8s_smoke Test Case: Creates a Guestbook application that contains redis + server, 2 instances of redis slave, frontend application, frontend service + and redis master service and redis slave service. Using frontend service, + the test will write an entry into the guestbook application which will + store the entry into the backend redis database. Application flow MUST + work as expected and the data written MUST be available to read. + + * Smoke Tier + + * k8s_conformance Test Case: Runs a series of k8s e2e tests expected to + pass on any Kubernetes cluster. It is a subset of tests necessary to + demonstrate conformance grows with each release. Conformance is thus + considered versioned, with backwards compatibility guarantees and are + designed to be run with no cloud provider configured. + + .. _`[2]`: http://docs.openstack.org/developer/tempest/overview.html .. _`[3]`: https://rally.readthedocs.org/en/latest/index.html .. _`[5]`: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater/blob/master/openstack-blueprint.yaml @@ -433,3 +460,4 @@ juju_epc .. _`[12]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/userguide/index.html .. _`[13]`: https://wiki.opnfv.org/display/PROJ/SNAPS-OO .. _`[14]`: https://github.com/oolorg/opnfv-functest-vrouter +.. _`[15]`: https://github.com/kubernetes/community/blob/master/contributors/devel/e2e-tests.md diff --git a/docs/testing/user/userguide/test_results.rst b/docs/testing/user/userguide/test_results.rst index 40a7af33c..bb28989ae 100644 --- a/docs/testing/user/userguide/test_results.rst +++ b/docs/testing/user/userguide/test_results.rst @@ -84,6 +84,42 @@ Parser testcase:: | parser-basics | parser | features | 00:00 | SKIP | +-----------------------+-----------------+------------------+------------------+----------------+ +Functest Kubernetes test result:: + + +--------------------------------------+------------------------------------------------------------+ + | ENV VAR | VALUE | + +--------------------------------------+------------------------------------------------------------+ + | INSTALLER_TYPE | compass | + | DEPLOY_SCENARIO | k8-nosdn-nofeature-ha | + | BUILD_TAG | jenkins-functest-compass-baremetal-daily-master-75 | + | CI_LOOP | daily | + +--------------------------------------+------------------------------------------------------------+ + +Kubernetes healthcheck suite:: + + +-------------------+------------------+---------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------+------------------+---------------------+------------------+----------------+ + | k8s_smoke | functest | healthcheck | 01:54 | PASS | + +-------------------+------------------+---------------------+------------------+----------------+ + +Kubernetes smoke suite:: + + +-------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------------+------------------+---------------+------------------+----------------+ + | k8s_conformance | functest | smoke | 57:47 | PASS | + +-------------------------+------------------+---------------+------------------+----------------+ + +Kubernetes features suite:: + + +----------------------+------------------+------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +----------------------+------------------+------------------+------------------+----------------+ + | stor4nfv_k8s | stor4nfv | stor4nfv | 00:00 | SKIP | + | clover_k8s | clover | clover | 00:00 | SKIP | + +----------------------+------------------+------------------+------------------+----------------+ + Results are automatically pushed to the test results database, some additional result files are pushed to OPNFV artifact web sites. diff --git a/functest/api/resources/v1/testcases.py b/functest/api/resources/v1/testcases.py index 2dbf97e4d..43340625b 100644 --- a/functest/api/resources/v1/testcases.py +++ b/functest/api/resources/v1/testcases.py @@ -146,12 +146,12 @@ class V1Testcase(ApiResource): """ Update the log file for each task""" rconfig = ConfigParser.RawConfigParser() rconfig.read( - pkg_resources.resource_filename('functest', 'ci/logging.ini')) + pkg_resources.resource_filename('xtesting', 'ci/logging.ini')) log_path = os.path.join(getattr(config.CONF, 'dir_results'), '{}.log'.format(task_id)) rconfig.set('handler_file', 'args', '("{}",)'.format(log_path)) with open( pkg_resources.resource_filename( - 'functest', 'ci/logging.ini'), 'wb') as configfile: + 'xtesting', 'ci/logging.ini'), 'wb') as configfile: rconfig.write(configfile) diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 2b775bca0..248086e15 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -44,7 +44,7 @@ class RallyBase(testcase.TestCase): """Base class form Rally testcases implementation.""" # pylint: disable=too-many-instance-attributes - TESTS = ['authenticate', 'glance', 'cinder', 'heat', + TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'vm', 'all'] GLANCE_IMAGE_NAME = getattr(config.CONF, 'openstack_image_name') GLANCE_IMAGE_FILENAME = getattr(config.CONF, 'openstack_image_file_name') diff --git a/functest/opnfv_tests/openstack/rally/scenario/full/opnfv-gnocchi.yaml b/functest/opnfv_tests/openstack/rally/scenario/full/opnfv-gnocchi.yaml new file mode 100644 index 000000000..8da5142e7 --- /dev/null +++ b/functest/opnfv_tests/openstack/rally/scenario/full/opnfv-gnocchi.yaml @@ -0,0 +1,163 @@ + Gnocchi.list_capabilities: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + Gnocchi.get_status: + - + args: + detailed: false + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicyRule.list_archive_policy_rule: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicyRule.create_archive_policy_rule: + - + args: + metric_pattern: "cpu_*" + archive_policy_name: "low" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicyRule.create_delete_archive_policy_rule: + - + args: + metric_pattern: "cpu_*" + archive_policy_name: "low" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicy.list_archive_policy: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicy.create_archive_policy: + - + args: + definition: + - granularity: "0:00:01" + timespan: "1:00:00" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicy.create_delete_archive_policy: + - + args: + definition: + - granularity: "0:00:01" + timespan: "1:00:00" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiResourceType.list_resource_type: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiResourceType.create_resource_type: + - + args: + attributes: + foo: + required: false + type: "string" + bar: + required: true + type: "number" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiResourceType.create_delete_resource_type: + - + args: + attributes: + foo: + required: false + type: "string" + bar: + required: true + type: "number" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiMetric.list_metric: + - + args: + limit: 10000 + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiMetric.create_metric: + - + args: + archive_policy_name: "low" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiMetric.create_delete_metric: + - + args: + archive_policy_name: "low" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} diff --git a/functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-gnocchi.yaml b/functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-gnocchi.yaml new file mode 100644 index 000000000..15185f385 --- /dev/null +++ b/functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-gnocchi.yaml @@ -0,0 +1,110 @@ + Gnocchi.list_capabilities: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + Gnocchi.get_status: + - + args: + detailed: false + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicyRule.list_archive_policy_rule: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicyRule.create_delete_archive_policy_rule: + - + args: + metric_pattern: "cpu_*" + archive_policy_name: "low" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicy.list_archive_policy: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiArchivePolicy.create_delete_archive_policy: + - + args: + definition: + - granularity: "0:00:01" + timespan: "1:00:00" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiResourceType.list_resource_type: + - + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiResourceType.create_delete_resource_type: + - + args: + attributes: + foo: + required: false + type: "string" + bar: + required: true + type: "number" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiMetric.list_metric: + - + args: + limit: 10000 + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + GnocchiMetric.create_delete_metric: + - + args: + archive_policy_name: "low" + context: + {{ user_context(tenants_amount, users_amount, use_existing_users) }} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} diff --git a/functest/opnfv_tests/openstack/rally/task.yaml b/functest/opnfv_tests/openstack/rally/task.yaml index 033edb831..fe9304fc2 100644 --- a/functest/opnfv_tests/openstack/rally/task.yaml +++ b/functest/opnfv_tests/openstack/rally/task.yaml @@ -15,6 +15,10 @@ {%- include "var/opnfv-cinder.yaml"-%} {% endif %} +{% if "gnocchi" in service_list %} +{%- include "var/opnfv-gnocchi.yaml"-%} +{% endif %} + {% if "keystone" in service_list %} {%- include "var/opnfv-keystone.yaml"-%} {% endif %} diff --git a/functest/opnfv_tests/vnf/epc/juju_epc.py b/functest/opnfv_tests/vnf/epc/juju_epc.py index a60d4f51e..1ba883d40 100644 --- a/functest/opnfv_tests/vnf/epc/juju_epc.py +++ b/functest/opnfv_tests/vnf/epc/juju_epc.py @@ -82,6 +82,8 @@ class JujuEpc(vnf.VnfOnBoarding): __logger = logging.getLogger(__name__) + juju_wait_timeout = '3600' + def __init__(self, **kwargs): if "case_name" not in kwargs: kwargs["case_name"] = "juju_epc" @@ -336,7 +338,7 @@ class JujuEpc(vnf.VnfOnBoarding): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self.__logger.info("%s\n%s", " ".join(cmd), output) self.__logger.info("Waiting for instances .....") - cmd = ['juju-wait'] + cmd = ['timeout', '-t', JujuEpc.juju_wait_timeout, 'juju-wait'] output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self.__logger.info("%s\n%s", " ".join(cmd), output) self.__logger.info("Deployed Abot-epc on Openstack") @@ -382,7 +384,7 @@ class JujuEpc(vnf.VnfOnBoarding): 'tagnames={}'.format(self.details['test_vnf']['tag_name'])] output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self.__logger.info("%s\n%s", " ".join(cmd), output) - cmd = ['juju-wait'] + cmd = ['timeout', '-t', JujuEpc.juju_wait_timeout, 'juju-wait'] output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self.__logger.info("%s\n%s", " ".join(cmd), output) duration = time.time() - start_time diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims_perf.yaml b/functest/opnfv_tests/vnf/ims/cloudify_ims_perf.yaml index 7573e079f..0e020cbe4 100644 --- a/functest/opnfv_tests/vnf/ims/cloudify_ims_perf.yaml +++ b/functest/opnfv_tests/vnf/ims/cloudify_ims_perf.yaml @@ -1,9 +1,9 @@ --- tenant_images: ubuntu_14.04: - http://cloud-images.ubuntu.com/trusty/current/ubuntu-14.04-server-cloudimg-amd64-disk1.img + /home/opnfv/functest/images/ubuntu-14.04-server-cloudimg-amd64-disk1.img cloudify_manager_4.0: - http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2 + /home/opnfv/functest/images/cloudify-manager-premium-4.0.1.qcow2 # PATH of the virtual chassis image. IXIA_CHASSIS: # Ixia_Virtual_Chassis_8.30_EA_KVM.qcow2 # PATH of the virtual card image. |