From 6ab2ccb7071f1d4925f25344d4b4a29d0525d935 Mon Sep 17 00:00:00 2001 From: Morgan Richomme Date: Mon, 8 Feb 2016 18:35:18 +0100 Subject: update doc - update external test cases - add chapter on push to DB Change-Id: I10cd5c9583233604c8b8595f173c690f45124789 Signed-off-by: Morgan Richomme --- docs/devguide/index.rst | 423 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 376 insertions(+), 47 deletions(-) (limited to 'docs/devguide') diff --git a/docs/devguide/index.rst b/docs/devguide/index.rst index bdf0e5b26..7dd5cc790 100644 --- a/docs/devguide/index.rst +++ b/docs/devguide/index.rst @@ -11,7 +11,8 @@ OPNFV FUNCTEST developer guide Introduction ============ -This document describes how feature projects aiming to run functional tests can be integrated into FuncTest framework. +This document describes how feature projects aiming to run functional tests can +be integrated into FuncTest framework. ================================ @@ -46,16 +47,25 @@ The Internal test cases in Brahmaputra are: * vIMS * Rally +The external tescases are: + + * Promise + * Doctor + * Onos + * BGPVPN + see `[2]`_ for details. -Functest can also be considered as a framework that may include external OPNFV projects. +Functest can also be considered as a framework that may include external OPNFV +projects. This framework will ease the integration of the feature test suite to the CI. ================== How Functest works ================== -The installation and the launch of the Functest docker image is described in `[1]`_. +The installation and the launch of the Functest docker image is described in +`[1]`_. The Functest docker directories are:: @@ -80,46 +90,50 @@ The Functest docker directories are:: :: - +--------------+-------------------+---------------------------------------------------------+ - | Directory | Subdirectory | Comments | - +--------------+-------------------+---------------------------------------------------------+ - | | /conf | All the useful configuration file(s) for | - | | | the openstack creds are put there for CI | - | | | It is recommended to push it there when passing the | - | | | credentials to container through the -v option | - | +-------------------+---------------------------------------------------------+ - | opnfv | /data | Usefull data, images for | - | | | By default we put an image cirros-0.3.4-x86_64-disk.img | - | | | This image can be used by any projects | - | +-------------------+---------------------------------------------------------+ - | | /results | Local result directory of project | - +--------------+-------------------+---------------------------------------------------------+ - | | bgpvpn | | - | +-------------------+ + - | repos | doctor | | - | +-------------------+ + - | | functest | | - | +-------------------+ + - | | odl_integration | | - | +-------------------+ Clone of the useful repositories + - | | onos | These repositories may include: | - | +-------------------+ - tooling + - | | promise | - scenario | - | +-------------------+ - scripts + - | | rally | | - | +-------------------+ + - | | releng | | - | +-------------------+ + - | | vims-test | | - | +-------------------+ + - | | | | - +--------------+-------------------+---------------------------------------------------------+ + +-----------+-------------------+---------------------------------------------+ + | Directory | Subdirectory | Comments | + +-----------+-------------------+---------------------------------------------+ + | | /conf | All the useful configuration file(s) for | + | | | the openstack creds are put there | + | | | for CI | + | | | It is recommended to push it there when | + | | | passing the credentials to container through| + | | | the -v option | + | +-------------------+---------------------------------------------+ + | opnfv | /data | Usefull data, images for | + | | | By default we put a cirros image: | + | | | cirros-0.3.4-x86_64-disk.img | + | | | This image can be used by any projects | + | +-------------------+---------------------------------------------+ + | | /results | Local result directory of project | + +-----------+-------------------+---------------------------------------------+ + | | bgpvpn | | + | +-------------------+ + + | repos | doctor | | + | +-------------------+ + + | | functest | | + | +-------------------+ + + | | odl_integration | | + | +-------------------+ Clone of the useful repositories + + | | onos | These repositories may include: | + | +-------------------+ - tooling + + | | promise | - scenario | + | +-------------------+ - scripts + + | | rally | | + | +-------------------+ + + | | releng | | + | +-------------------+ + + | | vims-test | | + | +-------------------+ + + | | | | + +-----------+-------------------+---------------------------------------------+ Before running the test suite, you must prepare the environement by running:: $ /home/opnfv/repos/functest/docker/prepare_env.sh -By running prepare_env.sh, you build the test environement required by the tests including the retrieval and sourcing of OpenStack credentials. +By running prepare_env.sh, you build the test environement required by the tests +including the retrieval and sourcing of OpenStack credentials. This is an example of the env variables we have in the docker container: * HOSTNAME=373f77816eb0 @@ -134,7 +148,8 @@ This is an example of the env variables we have in the docker container: * creds=/home/opnfv/functest/conf/openstack.creds * _=/usr/bin/env -The prepare_env.sh will source the credentials, so once run you should have access to the following env variables:: +The prepare_env.sh will source the credentials, so once run you should have +access to the following env variables:: root@373f77816eb0:~# env|grep OS_ OS_REGION_NAME=RegionOne @@ -173,7 +188,8 @@ The files of the Functest repository you must modify to integrate Functest are: Dockerfile ========== -This file lists the repositories to be cloned in the Functest container. The repositories can be internal or external:: +This file lists the repositories to be cloned in the Functest container. +The repositories can be internal or external:: RUN git clone https://gerrit.opnfv.org/gerrit/ ${repos_dir}/ @@ -181,7 +197,8 @@ This file lists the repositories to be cloned in the Functest container. The rep Common.sh ========== -This file can be used to declare the branch and commit variables of your projects:: +This file can be used to declare the branch and commit variables of your +projects:: _BRANCH=$(cat $config_file | grep -w _branch | awk 'END {print $NF}') _COMMIT=$(cat $config_file | grep -w _commit | awk 'END {print $NF}') @@ -193,7 +210,8 @@ This file can be used to declare the branch and commit variables of your projec requirements.pip ================ -This file can be used to preloaded all the needed Python libraries (and avoid that each project does it) +This file can be used to preloaded all the needed Python libraries (and avoid +that each project does it) The current libraries used in Functest container are:: # cat requirements.pip @@ -213,17 +231,22 @@ The current libraries used in Functest container are:: robotframework-sshlibrary==2.1.1 configObj==5.0.6 Flask==0.10.1 + xmltodict==0.9.2 + scp==0.10.2 + paramiko==1.16.0 prepare_env.sh ============== -This script can be adapted if you need to set up a specific environment before running the tests. +This script can be adapted if you need to set up a specific environment before +running the tests. run_tests.sh ============ -This script is used to run the tests. You must thus complete the cases with you own project:: +This script is used to run the tests. You must thus complete the cases with your +own project:: ;; "promise") @@ -248,7 +271,8 @@ And do not forget to update also the help line:: config_funtest.yaml =================== -This file is the main configuration file of Functest. You must add the references to your project:: +This file is the main configuration file of Functest. You must add the +references to your project:: general: directories: @@ -259,12 +283,317 @@ This file is the main configuration file of Functest. You must add the reference _commit: latest +==================== +Test Dashboard & API +==================== + +The OPNFV testing group created a test collection database to collect the test +results from CI. +Any test project running on any lab integrated in CI can push the results to +this database. +This database can be used afterwards to see the evolution of the tests and +compare the results versus the installers, the scenario or the labs. + +You can find more information about the dashboard from Testing Dashboard wiki +page `[3]`_. + + +Overall Architecture +-------------------- + +The Test result management in Brahmaputra can be summarized as follows:: + + +-------------+ +-------------+ +-------------+ + | | | | | | + | Test | | Test | | Test | + | Project #1 | | Project #2 | | Project #N | + | | | | | | + +-------------+ +-------------+ +-------------+ + | | | + V V V + +-----------------------------------------+ + | | + | Test Rest API front end | + | http://testresults.opnfv.org/testapi | + | | + +-----------------------------------------+ + A | + | V + | +-------------------------+ + | | | + | | Test Results DB | + | | Mongo DB | + | | | + | +-------------------------+ + | + | + +----------------------+ + | | + | test Dashboard | + | | + +----------------------+ + +The Test dashboard URL is: TODO LF +A alternate Test dashboard has been realized to provide a view per installer and +per scenario for Brahmaputra release:: + + http://testresults.opnfv.org/proto/brahma/ + +This Dashboard consumes the results retrieved thanks to the Test API. + +Test API description +-------------------- + +The Test API is used to declare pods, projects, test cases and test results. An +additional method dashboard has been added to post-process the raw results. The +data model is very basic, 4 objects are created: + + * Pods + * Test projects + * Test cases + * Test results + +Pods:: + + { + "id": , + "details": , + "creation_date": YYYY-MM-DD HH:MM:SS , + "name": , + "mode": + }, + +Test project:: + + { + "id": , + "name": , + "creation_date": "YYYY-MM-DD HH:MM:SS", + "description": + }, + +Test case:: + + { + "id": , + "name":, + "creation_date": "YYYY-MM-DD HH:MM:SS", + "description": , + "url": + }, + +Test results:: + + { + "_id": , + "pod_name": , + "version": , + "installer": , + "description": , + "creation_date": "YYYY-MM-DD HH:MM:SS", + "case_name": + "details":{ + <- the results to be put here -> + } + + +For Brahmaputra, we got: + + * 16 pods + * 18 projects + * 101 test cases + +The projects and the test cases have been frozen in December. +But all were not ready for Brahmaputra. + + + +The API can described as follows: + +Version: + + +--------+--------------------------+-----------------------------------------+ + | Method | Path | Description | + +========+==========================+=========================================+ + | GET | /version | Get API version | + +--------+--------------------------+-----------------------------------------+ + + +Pods: + + +--------+--------------------------+-----------------------------------------+ + | Method | Path | Description | + +========+==========================+=========================================+ + | GET | /pods | Get the list of declared Labs (PODs) | + +--------+--------------------------+-----------------------------------------+ + | POST | /pods | Declare a new POD | + | | | Content-Type: application/json | + | | | { | + | | | "name": "pod_foo", | + | | | "creation_date": "YYYY-MM-DD HH:MM:SS"| + | | | } | + +--------+--------------------------+-----------------------------------------+ + +Projects: + + +--------+--------------------------+-----------------------------------------+ + | Method | Path | Description | + +========+==========================+=========================================+ + | GET | /test_projects | Get the list of test projects | + +--------+--------------------------+-----------------------------------------+ + | GET |/test_projects/{project} | Get details on {project} | + | | | | + +--------+--------------------------+-----------------------------------------+ + | POST | /test_projects | Add a new test project | + | | | Content-Type: application/json | + | | | { | + | | | "name": "project_foo", | + | | | "description": "whatever you want" | + | | | } | + +--------+--------------------------+-----------------------------------------+ + | PUT | /test_projects/{project} | Update a test project | + | | | | + | | | Content-Type: application/json | + | | | { | + | | | | + | | | } | + +--------+--------------------------+-----------------------------------------+ + | DELETE | /test_projects/{project} | Delete a test project | + +--------+--------------------------+-----------------------------------------+ + + +Test cases: + + +--------+--------------------------+-----------------------------------------+ + | Method | Path | Description | + +========+==========================+=========================================+ + | GET | /test_projects/{project}/| Get the list of test cases of {project} | + | | cases | | + +--------+--------------------------+-----------------------------------------+ + | POST | /test_projects/{project}/| Add a new test case to {project} | + | | cases | Content-Type: application/json | + | | | { | + | | | "name": "case_foo", | + | | | "description": "whatever you want" | + | | | "creation_date": "YYYY-MM-DD HH:MM:SS"| + | | | "url": "whatever you want" | + | | | } | + +--------+--------------------------+-----------------------------------------+ + | PUT | /test_projects/{project}?| Modify a test case of {project} | + | | case_name={case} | | + | | | Content-Type: application/json | + | | | { | + | | | | + | | | } | + +--------+--------------------------+-----------------------------------------+ + | DELETE | /test_projects/{project}/| Delete a test case | + | | case_name={case} | | + +--------+--------------------------+-----------------------------------------+ + +Test Results: + + +--------+--------------------------+-----------------------------------------+ + | Method | Path | Description | + +========+==========================+=========================================+ + | GET |/results/project={project}| Get the test results of {project} | + +--------+--------------------------+-----------------------------------------+ + | GET |/results/case={case} | Get the test results of {case} | + +--------+--------------------------+-----------------------------------------+ + | GET |/results?pod={pod} | get the results on pod {pod} | + +--------+--------------------------+-----------------------------------------+ + | GET |/results?installer={inst} | Get the test results of installer {inst}| + +--------+--------------------------+-----------------------------------------+ + | GET |/results?version={version}| Get the test results of scenario | + | | | {version}. Initially the version param | + | | | was reflecting git version, in Functest | + | | | it was decided to move to scenario | + +--------+--------------------------+-----------------------------------------+ + | GET |/results?project={project}| Get all the results of the test case | + | |&case={case} | {case} of the project {project} with | + | |&version={scenario} | version {scenario} installed by | + | |&installer={installer} | {installer} on POD {pod} stored since | + | |&pod={pod} | {days} days | + | | | {project_name} and {case_name} are | + | |&period={days} | mandatory, the other parameters are | + | | | optional. | + +--------+--------------------------+-----------------------------------------+ + | POST | /results | Add a new test results | + | | | Content-Type: application/json | + | | | { | + | | | "project_name": "project_foo", | + | | | "case_name": "case_foo", | + | | | "pod_name": "pod_foo", | + | | | "installer": "installer_foo", | + | | | "version": "scenario_foo", | + | | | "details": | + | | | } | + +--------+--------------------------+-----------------------------------------+ + + +Dashboard: + + +--------+--------------------------+-----------------------------------------+ + | Method | Path | Description | + +========+==========================+=========================================+ + | GET |/dashboard? | Get all the dashboard ready results of | + | |&project={project} | {case} of the project {project} | + | |&case={case} | version {scenario} installed by | + | |&version={scenario} | {installer} on POD {pod} stored since | + | |&installer={installer} | {days} days | + | |&pod={pod} | | + | |&period={days} | {project_name} and {case_name} are | + | | | mandatory, the other parameters are | + | | | optional. | + +--------+--------------------------+-----------------------------------------+ + + +The results with dashboard method are post-processed from raw results. +Please note that dashboard results are not stored. Only raw results are stored. + + +=============================================== +How to push your results into the Test Database +=============================================== + +The test database is used to collect test results. By default it is enabled only +in Continuous Integration. +The architecture and associated API is described in `[2]`_. +If you want to push your results from CI, you just have to use the API at the +end of your script. + +You can also reuse a python function defined in functest_utils.py:: + + def push_results_to_db(db_url, case_name, logger, pod_name,version, payload): + """ + POST results to the Result target DB + """ + url = db_url + "/results" + installer = get_installer_type(logger) + params = {"project_name": "functest", "case_name": case_name, + "pod_name": pod_name, "installer": installer, + "version": version, "details": payload} + + headers = {'Content-Type': 'application/json'} + try: + r = requests.post(url, data=json.dumps(params), headers=headers) + if logger: + logger.debug(r) + return True + except Exception, e: + print "Error [push_results_to_db('%s', '%s', '%s', '%s', '%s')]:" \ + % (db_url, case_name, pod_name, version, payload), e + return False + +:: + ========== References ========== -.. _`[1]`: Functest configuration guide URL -.. _`[2]`: functest user guide URL +.. _`[1]`: http://artifacts.opnfv.org/functest/docs/configguide/index.html Functest configuration guide URL +.. _`[2]`: http://artifacts.opnfv.org/functest/docs/userguide/index.html functest user guide URL +.. _`[3]`: https://wiki.opnfv.org/opnfv_test_dashboard OPNFV main site: opnfvmain_. -- cgit 1.2.3-korg