diff options
Diffstat (limited to 'docs/testing')
25 files changed, 421 insertions, 2565 deletions
diff --git a/docs/testing/developer/devguide/conf.py b/docs/testing/developer/devguide/conf.py deleted file mode 100644 index 28255a69e..000000000 --- a/docs/testing/developer/devguide/conf.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Functest Developer Guide documentation build configuration file, created by -# sphinx-quickstart on Wed Apr 4 05:36:49 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_opnfv_theme - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Functest Developer Guide' -copyright = u'2018, Functest <opnfv-tech-discuss@lists.opnfv.org>' -author = u'Functest <opnfv-tech-discuss@lists.opnfv.org>' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'opnfv' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = { - 'bootswatch_theme': 'journal', - 'navbar_sidebarrel': False, - 'navbar_title': '', -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] -html_theme_path = sphinx_opnfv_theme.get_html_theme_path() - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'FunctestDeveloperGuidedoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, - 'FunctestDeveloperGuide.tex', - u'Functest Developer Guide Documentation', - u'Functest \\textless{}opnfv-tech-discuss@lists.opnfv.org\\textgreater{}', - 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, - 'functestdeveloperguide', - u'Functest Developer Guide Documentation', - [author], - 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, - 'FunctestDeveloperGuide', - u'Functest Developer Guide Documentation', - author, - 'FunctestDeveloperGuide', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/docs/testing/developer/devguide/index.rst b/docs/testing/developer/devguide/index.rst index caa1ccb3b..da5485ef0 100644 --- a/docs/testing/developer/devguide/index.rst +++ b/docs/testing/developer/devguide/index.rst @@ -5,7 +5,6 @@ Functest Developer Guide ************************ .. toctree:: - :numbered: :maxdepth: 2 ============ @@ -53,8 +52,6 @@ to the different tiers: * functest-core: https://hub.docker.com/r/opnfv/functest-core/ * functest-healthcheck: https://hub.docker.com/r/opnfv/functest-healthcheck/ * functest-smoke: https://hub.docker.com/r/opnfv/functest-smoke/ - * functest-features: https://hub.docker.com/r/opnfv/functest-features/ - * functest-components: https://hub.docker.com/r/opnfv/functest-components/ * functest-vnf: https://hub.docker.com/r/opnfv/functest-vnf/ * functest-restapi: https://hub.docker.com/r/opnfv/functest-restapi/ @@ -82,9 +79,7 @@ Functest internal test cases The internal test cases in Euphrates are: - * api_check * connection_check - * snaps_health_check * vping_ssh * vping_userdata * odl @@ -106,49 +101,12 @@ The main internal test cases are in the opnfv_tests subfolder of the repository, the internal test cases can be grouped by domain: * sdn: odl, odl_fds - * openstack: api_check, connection_check, snaps_health_check, vping_ssh, - vping_userdata, tempest_*, rally_* + * openstack: connection_check, vping_ssh, vping_userdata, tempest_*, rally_* * vnf: cloudify_ims If you want to create a new test case you will have to create a new folder under the testcases directory (See next section for details). -Functest external test cases -============================ -The external test cases are inherited from other OPNFV projects, especially the -feature projects. - -The external test cases are: - - * barometer - * bgpvpn - * doctor - * domino - * fds - * promise - * refstack_defcore - * snaps_smoke - * functest-odl-sfc - * orchestra_clearwaterims - * orchestra_openims - * vyos_vrouter - * juju_vepc - -External test cases integrated in previous versions but not released in -Euphrates: - - * copper - * moon - * netready - * security_scan - - -The code to run these test cases is hosted in the repository of the project. -Please note that orchestra test cases are hosted in Functest repository and not -in orchestra repository. Vyos_vrouter and juju_vepc code is also hosted in -functest as there are no dedicated projects. - - Functest framework ================== @@ -168,8 +126,8 @@ The definition of the tiers has been agreed by the testing working group. The tiers are: * healthcheck * smoke + * benchmarking * features - * components * vnf Functest abstraction classes @@ -211,35 +169,6 @@ Feature, unit and vnf_base inherit from testcase:: +--------------------+ +---------+ +------------------------+ +-----------------+ -Testcase --------- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.testcase.html - -Feature -------- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.feature.html - -Unit ----- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.unit.html - -VNF ---- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.vnf.html - -Robotframework --------------- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.robotframework.html - - -see `[5]`_ to get code samples - - Functest util classes ===================== @@ -259,14 +188,11 @@ as follows:: |-- openstack_tacker.py `-- openstack_utils.py -It is recommended to use the SNAPS-OO library for deploying OpenStack -instances. SNAPS `[4]`_ is an OPNFV project providing OpenStack utils. - TestAPI ======= Functest is using the Test collection framework and the TestAPI developed by -the OPNFV community. See `[6]`_ for details. +the OPNFV community. See `[4]`_ for details. Reporting @@ -278,17 +204,10 @@ jinja2 templates `[3]`_. Dashboard ========= -Additional dashboarding is managed at the testing group level, see `[7]`_ for +Additional dashboarding is managed at the testing group level, see `[5]`_ for details. -======= -How TOs -======= - -See How to section on Functest wiki `[8]`_ - - ========== References ========== @@ -297,16 +216,10 @@ _`[1]`: http://artifacts.opnfv.org/functest/docs/configguide/index.html Functest _`[2]`: http://artifacts.opnfv.org/functest/docs/userguide/index.html functest user guide -_`[3]`: https://git.opnfv.org/cgit/releng/tree/utils/test/reporting - -_`[4]`: https://git.opnfv.org/snaps/ - -_`[5]` : http://testresults.opnfv.org/functest/framework/index.html - -_`[6]`: http://docs.opnfv.org/en/latest/testing/testing-dev.html +_`[3]`: https://github.com/opnfv/releng-testresults/tree/master/reporting -_`[7]`: https://opnfv.biterg.io/goto/283dba93ca18e95964f852c63af1d1ba +_`[4]`: https://wiki.opnfv.org/display/functest/2017+Beijing?preview=%2F11699623%2F11700523%2FTestAPI+-+test+results+collection+service.pptx -_`[8]`: https://wiki.opnfv.org/pages/viewpage.action?pageId=7768932 +_`[5]`: https://lfanalytics.io/projects/lfn%2Fopnfv/dashboard IRC support chan: #opnfv-functest diff --git a/docs/testing/developer/internship/security_group/conf.py b/docs/testing/developer/internship/security_group/conf.py deleted file mode 100644 index 46080c6d6..000000000 --- a/docs/testing/developer/internship/security_group/conf.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Functest Security group test cases documentation build configuration file, -# created by sphinx-quickstart on Wed Apr 4 05:56:19 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_opnfv_theme - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Functest Security group test cases' -copyright = u'2018, Functest <opnfv-tech-discuss@lists.opnfv.org>' -author = u'Functest <opnfv-tech-discuss@lists.opnfv.org>' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'opnfv' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = { - 'bootswatch_theme': 'journal', - 'navbar_sidebarrel': False, - 'navbar_title': '', -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] -html_theme_path = sphinx_opnfv_theme.get_html_theme_path() - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'FunctestSecuritygrouptestcasesdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, - 'FunctestSecuritygrouptestcases.tex', - u'Functest Security group test cases Documentation', - u'Functest \\textless{}opnfv-tech-discuss@lists.opnfv.org\\textgreater{}', - 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, - 'functestsecuritygrouptestcases', - u'Functest Security group test cases Documentation', - [author], - 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, - 'FunctestSecuritygrouptestcases', - u'Functest Security group test cases Documentation', - author, - 'FunctestSecuritygrouptestcases', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/docs/testing/developer/internship/security_group/index.rst b/docs/testing/developer/internship/security_group/index.rst deleted file mode 100644 index 87bcd90e5..000000000 --- a/docs/testing/developer/internship/security_group/index.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-4.0 - -================================== -Functest Security group test cases -================================== - -Author: Girish Sukhatankar -mentors: D.Blaisonneau, J.Lausuch, M.Richomme - -Abstract -======== - - -Version history -=============== - -+------------+----------+------------------+------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+------------------------+ -| 2016-??-?? | 0.0.1 | Morgan Richomme | Beginning of the | -| | | (Orange) | Internship | -+------------+----------+------------------+------------------------+ - - -Overview: -========= - - - - -Problem Statement: ------------------- - - - -Curation Phase --------------- - - - - - -Schedule: -========= - - - -+--------------------------+------------------------------------------+ -| **Date** | **Comment** | -| | | -+--------------------------+------------------------------------------+ -| December - January | '........' | -+--------------------------+------------------------------------------+ -| January - february | '........' | -+--------------------------+------------------------------------------+ - - -References: -=========== - -.. _`[1]` : https://wiki.opnfv.org/display/DEV/Intern+Project%3A+Security+groups+test+case+in+Functest diff --git a/docs/testing/developer/internship/testapi_evolution/conf.py b/docs/testing/developer/internship/testapi_evolution/conf.py deleted file mode 100644 index f59308c59..000000000 --- a/docs/testing/developer/internship/testapi_evolution/conf.py +++ /dev/null @@ -1,186 +0,0 @@ -# -*- coding: utf-8 -*- -# -# TestAPI evolution documentation build configuration file, created by -# sphinx-quickstart on Wed Apr 4 06:00:42 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_opnfv_theme - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'TestAPI evolution' -copyright = u'2018, Functest <opnfv-tech-discuss@lists.opnfv.org>' -author = u'Functest <opnfv-tech-discuss@lists.opnfv.org>' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'opnfv' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = { - 'bootswatch_theme': 'journal', - 'navbar_sidebarrel': False, - 'navbar_title': '', -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] -html_theme_path = sphinx_opnfv_theme.get_html_theme_path() - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'TestAPIevolutiondoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, - 'TestAPIevolution.tex', - u'TestAPI evolution Documentation', - u'Functest \\textless{}opnfv-tech-discuss@lists.opnfv.org\\textgreater{}', - 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, - 'testapievolution', - u'TestAPI evolution Documentation', - [author], - 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, - 'TestAPIevolution', - u'TestAPI evolution Documentation', - author, - 'TestAPIevolution', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/docs/testing/developer/internship/testapi_evolution/index.rst b/docs/testing/developer/internship/testapi_evolution/index.rst deleted file mode 100644 index d2704de53..000000000 --- a/docs/testing/developer/internship/testapi_evolution/index.rst +++ /dev/null @@ -1,230 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-4.0 - -================= -TestAPI evolution -================= - -Author: Sakala Venkata Krishna Rohit -Mentors: S. Feng, J.Lausuch, M.Richomme - -Abstract -======== - -The TestAPI is used by all the test opnfv projects to report results. -It is also used to declare projects, test cases and labs. A major refactoring -has been done in Colorado with the introduction of swagger. The TestAPI is defined in Functest -developer guide. The purpose of this project is to add more features to the TestAPI that automate -the tasks that are done manually now, though there are tasks other than automation. - -Version history -=============== - -+------------+----------+------------------+------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+------------------------+ -| 2016-11-14 | 0.0.1 | Morgan Richomme | Beginning of the | -| | | (Orange) | Internship | -+------------+----------+------------------+------------------------+ -| 2017-02-17 | 0.0.2 | S.V.K Rohit | End of the Internship | -| | | (IIIT Hyderabad) | | -+------------+----------+------------------+------------------------+ - -Overview: -========= - -The internhip time period was from Nov 14th to Feb 17th. The project prosposal page is here `[1]`_. -The intern project was assigned to Svk Rohit and was mentored by S. Feng, J.Lausuch, M.Richomme. -The link to the patches submitted is `[2]`_. The internship was successfully completed and the -documentation is as follows. - -Problem Statement: ------------------- - -The problem statement could be divided into pending features that needed to be added into TestAPI -repo. The following were to be accomplished within the internship time frame. - -* **Add verification jenkins job for the TestAPI code** - The purpose of this job is to verify whehter the unit tests are successful or not with the - inclusion of the patchset submitted. - -* **Automatic update of opnfv/testapi docker image** - The docker image of TestAPI is hosted in the opnfv docker hub. To ensure that the TestAPI image - is always updated with the repository, automatic updation of the image is necessary and a job - is triggered whenever a new patch gets merged. - -* **Automation deployment of testresults.opnfv.org/test/ website** - In the same manner as the docker image of TestAPI is updated, the TestAPI website needs to be - in sync with the repository code. So, a job has been added to the opnfv jenkins ci for the - updation of the testresults website. - -* **Generate static documentation of TestAPI calls** - The purpose of this is to give an static/offline view of TestAPI. If someone wants to have a - look at the Restful APIs of TestAPI, he/she does't need to go to the website, he can download - a html page and view it anytime. - -* **Backup MongoDB of TestAPI** - The mongoDB needs to be backed up every week. Till now it was done manually, but due to this - internship, it is now automated using a jenkins job. - -* **Add token based authorization to the TestAPI calls** - The token based authorization was implemented to ensure that only ci_pods could access the - database. Authentication has been added to only delete/put/post requests. - -Curation Phase: ---------------- - -The curation phase was the first 3 to 4 weeks of the internship. This phase was to get familiar -with the TestAPI code and functionality and propose the solutions/tools for the tasks mentioned -above. Swagger codegen was choosen out of the four tools proposed `[3]`_ for generating static -documentaion. - -Also, specific amount of time was spent on the script flow of the jenkins jobs. The automatic -deployment task involves accessing a remote server from inside the jenkins build. The deployment -had to be done only after the docker image update is done. For these constraints to satisfy, a -multijob jenkins job was choosen instead of a freestyle job. - -Important Links: ----------------- - -* MongoDB Backup Link - `[4]`_ -* Static Documentation - `[5]`_ -* TestAPI Token addition to ci_pods - `[6]`_ - -Schedule: -========= - -The progress and completion of the tasks is described in the below table. - -+--------------------------+------------------------------------------+ -| **Date** | **Comment** | -| | | -+--------------------------+------------------------------------------+ -| Nov 14th - Dec 31st | Understand TestAPI code and the | -| | requirements. | -+--------------------------+------------------------------------------+ -| Jan 1st - Jan 7th | Add jenkins job to create static | -| | documentation and write build scripts. | -+--------------------------+------------------------------------------+ -| Jan 8th - Jan 21st | Add verification jenkins job for unit | -| | tests. | -+--------------------------+------------------------------------------+ -| Jan 22nd - Jan 28th | Add jenkins job for mongodb backup | -| | | -+--------------------------+------------------------------------------+ -| Jan 29th - Feb 11th | Enable automatic deployment of | -| | testresults.opnfv.org/test/ | -+--------------------------+------------------------------------------+ -| Feb 12th - Feb 17th | Add token based authentication | -| | | -+--------------------------+------------------------------------------+ - -FAQ's -===== - -This section lists the problems that I have faced and the understanding that I have acquired during -the internship. This section may help other developers in solving any errors casused because of the -code written as a part of this internship. - - -TestAPI -------- - -What is the difference between defining data_file as "/etc/.." and "etc/.." in setup.cfg ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If in the setup.cfg, it is defined as - -[files] -data_files = -etc/a.conf = etc/a.conf.sample - -then it ends up installed in the /usr/etc/. With this configuration, it would be installed -correctly within a venv. but when it is defined as - -[files] -data_files = -/etc/a.conf = etc/a.conf.sample - -then it ends up installed on the root of the filesystem instead of properly be installed within the -venv. - -Which attribute does swagger-codegen uses as the title in the generation of document generation ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It uses the nickname of the api call in swagger as the title in the generation of the document -generation. - -Does swagger-codegen take more than one yaml file as input ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -No, swagger-codegen only takes one yaml file as input to its jar file. If there more than one yaml -file, one needs to merge them and give it as an input keeping mind the swagger specs. - - -Jenkins & JJB -------------- - -Which scm macro is used for verification jenkins jobs ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are two macros for scm one is git-scm and other git-scm-gerrit. git-scm-gerrit is used for -verification jenkins job. - -Does the virtualenv created in one build script exists in other build scripts too ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -No, the virtualenv created in one build script only exists in that build script/shell. - -What parameters are needed for the scm macros ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Project and Branch are the two parameters needed for scm macros. - -What is the directory inside the jenkins build ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The directory of the jenkins build is the directory of the repo. `ls $WORKSPACE` command will give -you all the contents of the directory. - -How to include a bash script in jenkins job yaml file ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -An example might be apt here as an answer. - -builders: - - shell: - !include-raw: include-raw001-hello-world.sh - - -How do you make a build server run on a specific machine ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It can be done by defining a label parameter 'SLAVE_LABEL' or in OPNFV , there are macros for each -server, one can use those parameter macros. -Ex: opnfv-build-defaults. Note, if we use macro, then no need to define GIT_BASE, but if one uses -SLAVE_LABEL, one needs to define a parameter GIT_BASE. This is because macro already has GIT_BASE -defined. - -What job style should be used when there is a situation like one build should trigger other builds -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -or when different build scripts need to be run on different machines ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -MultiJob style should be used as it has phases where each phase can be taken as a build scipt and -can have its own parameters by which one can define the SLAVE_LABEL parameter. - -References: -=========== - -_`[1]` : https://wiki.opnfv.org/display/DEV/Intern+Project%3A+testapi+evolution - -_`[2]` : https://gerrit.opnfv.org/gerrit/#/q/status:merged+owner:%22Rohit+Sakala+%253Crohitsakala%2540gmail.com%253E%22 - -_`[3]` : https://docs.google.com/document/d/1jWwVZ1ZpKgKcOS_zSz2KzX1nwg4BXxzBxcwkesl7krw/edit?usp=sharing - -_`[4]` : http://artifacts.opnfv.org/testapibackup.html - -_`[5]` : http://artifacts.opnfv.org/releng/docs/testapi.html - -_`[6]` : http://artifacts.opnfv.org/functest/docs/devguide/index.html#test-api-authorization diff --git a/docs/testing/developer/internship/unit_tests/conf.py b/docs/testing/developer/internship/unit_tests/conf.py deleted file mode 100644 index 00dec3c82..000000000 --- a/docs/testing/developer/internship/unit_tests/conf.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Functest Unit tests documentation build configuration file, created by -# sphinx-quickstart on Wed Apr 4 06:04:23 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_opnfv_theme - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Functest Unit tests' -copyright = u'2018, Functest <opnfv-tech-discuss@lists.opnfv.org>' -author = u'Functest <opnfv-tech-discuss@lists.opnfv.org>' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'opnfv' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = { - 'bootswatch_theme': 'journal', - 'navbar_sidebarrel': False, - 'navbar_title': '', -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] -html_theme_path = sphinx_opnfv_theme.get_html_theme_path() - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'FunctestUnittestsdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, - 'FunctestUnittests.tex', - u'Functest Unit tests Documentation', - u'Functest \\textless{}opnfv-tech-discuss@lists.opnfv.org\\textgreater{}', - 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, - 'functestunittests', - u'Functest Unit tests Documentation', - [author], - 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, - 'FunctestUnittests', - u'Functest Unit tests Documentation', - author, - 'FunctestUnittests', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/docs/testing/developer/internship/unit_tests/index.rst b/docs/testing/developer/internship/unit_tests/index.rst deleted file mode 100644 index f79e9e296..000000000 --- a/docs/testing/developer/internship/unit_tests/index.rst +++ /dev/null @@ -1,123 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-4.0 - -=================== -Functest Unit tests -=================== - -Author: Ashish Kumar -Mentors: H.Yao, J.Lausuch, M.Richomme - -Abstract -======== - - -Version history -=============== - -+------------+----------+------------------+------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+------------------------+ -| 2016-11-14 | 0.0.1 | Morgan Richomme | Beginning of the | -| | | (Orange) | Internship | -+------------+----------+------------------+------------------------+ -| 2017-03-31 | 0.0.2 | Ashish Kumar | During the | -| | | (IIIT Hyderabad) | Internship | -+------------+----------+------------------+------------------------+ - - -Overview: -========= -Functest project is developing and integrating functional test cases for OPNFV -and it is part of OPNFV since the beginning. Functest develops its own -testcases and framework. This framework includes several utility libraries. The -Project is growing rapidly with more features, tests added as per requirement. -It becomes the responsibility of every developer to maintain the integrity of -code i.e. new patch should not break the previous functionality of the project. -To automate this process of software development, we should write unit tests -and add them to CI so that when a new patch is ready to merge, we shouldn't -allow those which are breaking previous unit tests or decreasing the coverage. - - - -Problem Statement: ------------------- -The goal of the intership consists in creating unit test suites on Functest -code with good code coverage (>80%) and integrate it in continuous integration -in order to consolidate existing code. - - -Curation Phase --------------- -The curation phase was the first 3 to 4 weeks of the internship. This phase was -to get familiar with the functest code and functionality and explore the -solutions for unit testing in other projects and come up with the strategy for -writing unit tests in functest. - -In this phase we decided, - - - Coverage should be 80%. There are some functions like __init__, getter, - setter and other private methods for which writing unit test is a tedious - job, so we are leaving these methods for now. - - Do method wise testing for every module. - - Use mock for external or third party services, system calls and other - external library calls which could impact the behaviour of system during the - run of unit test. - - Add it in jenkins as passing criteria for patches. - - Write tests in modular way so that it can help to serve as a form of - documentation. - - - -Schedule: -========= -+--------------------------+------------------------------------------+ -| **Date** | **Comment** | -| | | -+--------------------------+------------------------------------------+ -| Nov 14th - Nov 28th | 1. Learn Functest Project Business | -| | 2. Set up the development environment | -| | 3. Run Functest code | -+--------------------------+------------------------------------------+ -| Nov 28th - Dec.9th | 1. Explore Unit Testing Strategy, | -| | 2. Learn about Mock in python | -+--------------------------+------------------------------------------+ -| Dec 12th - Dec 23rd | Implement Unit Tests for CLI | -| | | -+--------------------------+------------------------------------------+ -| Dec 26th - Jan 6th | Implement Unit Tests for Utils | -| | | -+--------------------------+------------------------------------------+ -| Jan 9th - Jan 20th | Implement Unit Tests for CI | -| | | -+--------------------------+------------------------------------------+ -| Jan 23rd - Feb 3rd | Implement Unit Tests for Core | -| | | -+--------------------------+------------------------------------------+ -| Feb 6th - Feb 17th | Implement Unit Tests for | -| | opnfv_tests/openstack/tempest | -+--------------------------+------------------------------------------+ -| Feb 20th - Mar 3rd | Implement Unit Tests for | -| | opnfv_tests/openstack/rally | -+--------------------------+------------------------------------------+ -| Mar 6th - Mar 17th | Implement Unit Tests for | -| | opnfv_tests/vnf/ims | -+--------------------------+------------------------------------------+ -| Mar 20th - Mar 31st | Recheck and Increase Coverage for all | -| | modules > 80% | -+--------------------------+------------------------------------------+ -| Apr 3rd - Apr 14th | Add CI Gating for unit tests | -| | | -+--------------------------+------------------------------------------+ -| Apr 17th - Apr 28th | Use Tox Utility, Documentation | -| | | -+--------------------------+------------------------------------------+ -| Apr 28th - End | Bug Fixing | -| | | -+--------------------------+------------------------------------------+ - - -References: -=========== - -.. _`[1]` : https://wiki.opnfv.org/display/DEV/Intern+Project%3A+Functest+unit+tests diff --git a/docs/testing/developer/internship/vnf_catalog/conf.py b/docs/testing/developer/internship/vnf_catalog/conf.py deleted file mode 100644 index de8336225..000000000 --- a/docs/testing/developer/internship/vnf_catalog/conf.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Open Source VNF Catalog documentation build configuration file, created by -# sphinx-quickstart on Wed Apr 4 06:07:56 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_opnfv_theme - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Open Source VNF Catalog' -copyright = u'2018, Functest <opnfv-tech-discuss@lists.opnfv.org>' -author = u'Functest <opnfv-tech-discuss@lists.opnfv.org>' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'opnfv' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = { - 'bootswatch_theme': 'journal', - 'navbar_sidebarrel': False, - 'navbar_title': '', -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] -html_theme_path = sphinx_opnfv_theme.get_html_theme_path() - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'OpenSourceVNFCatalogdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, - 'OpenSourceVNFCatalog.tex', - u'Open Source VNF Catalog Documentation', - u'Functest \\textless{}opnfv-tech-discuss@lists.opnfv.org\\textgreater{}', - 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, - 'opensourcevnfcatalog', - u'Open Source VNF Catalog Documentation', - [author], - 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, - 'OpenSourceVNFCatalog', - u'Open Source VNF Catalog Documentation', - author, - 'OpenSourceVNFCatalog', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/docs/testing/developer/internship/vnf_catalog/index.rst b/docs/testing/developer/internship/vnf_catalog/index.rst deleted file mode 100644 index 634b728bd..000000000 --- a/docs/testing/developer/internship/vnf_catalog/index.rst +++ /dev/null @@ -1,160 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-4.0 - -======================= -Open Source VNF Catalog -======================= - -Author: Kumar Rishabh -Mentors: B.Souville, M.Richomme, J.Lausuch - -Abstract -======== - - - -Version hissory -=============== - -+------------+----------+------------------+------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+------------------------+ -| 2016-12-12 | 0.0.1 | Morgan Richomme | Beginning of the | -| | | (Orange) | Internship | -+------------+----------+------------------+------------------------+ - - -Overview: -========= - - -This project aims to create an Open Source catalog for reference and -classification of Virtual Network Functions (VNFs)s available on -Internet. The classification method proposed will be in sync with the -requirements of Telcos active in NFV landscape. The project aims to have -running web platform similar to [1] by the mid of internship (2nd week -of March). By the penultimate month of internship I aim to have fully -functional implementation of an Open Source VNF in functest. - - -Problem Statement: ------------------- - -OPNFV aims to be the reference platform for development, -standardization and integration of Open Source NFV components across -various Open Source Platforms. It mainly deals with the infrastructure -through the Network Function Virtualization Infrastructure (NFVI) and -Virtual Infrastructure manager (VIM). The MANO (Management and -orchestration) stacks have been introduced recently. VNFs are not -directly in OPNFV scope, however VNFs are needed to test and qualify the -infrastructure. In this regard having a common curated Open Source -Reference VNF catalog would be of immense importance to community. - -Since major focus of OPNFV is Telcos, a curated platform targeted from -industry point of view would be very useful. We plan to divide the -entire project into three major phases(with some iterative improvements -and overlaps) - - -Curation Phase --------------- -This phase pertains to studying various Open Source VNFs available and -classification of them based on certain parameters. The parameters that -I currently have in mind are: - - * Developer Metrics: These pertain to repo characteristics of VNF under - study - * Usage Statistics - Activity, Number of Commits, stars - * Maturity Statistics - For instance if an NFV community decides code - coverage is important for them, it shows the NFV community is serious - about taking the project forward - * Technical Tagging: These are the tags that pertain to technical - characteristics of a VNF - * Broad Use Cases - Whether the VNF fits strictly in IaaS, PaaS or - SaaS layer or is an hybrid of two/all. - * Generic Use Cases - This in my opinion is the broadest - classification category. For instance a VNF could be built with a - broad idea of powering IOT devices at home or from usage perspective - of Telco Operators (vFW, vEPC, vIMS, vCDN, vAAA, vCPE,...).`[2]`_ - * Fields of Application - * Library Status - Whether APIs are standardized, support RESTful - services. - * Dependency Forwarding Graph - This is pretty complex tagging - mechanism. It essentially tries to establish a graph relationship - between the VNFs (elementary VNFs are used in Service Function - Chaining chains such as Firewall, DPI, content enrichment,..). In my - opinion this is useful immensely. This will allow users to go to - platform and ask a question like - “I have this X tech stack to - support, Y and Z are my use cases, which NFVs should I use to support - this. - * Visitor Score - Based on `[1]`_ I plan to evolve a visitor score for - the platform. This will allow users to score an NFV on certain - parameters, may be post comments. - -**I plan to use the above three scores and evolve cumulative score which -will be displayed next to each of the NFV on the platform.** - - * Platform building phase - This will involve erecting a Web Platform - which will be similar to this `[1]`_. I am decently familiar with - Django and hence I will write the platform in Django. There are two - action plans that I have in mind right now. Either I can start writing - the platform simultaneously which will help keep track of my progress - or I can write the platform after 1.5 - 2 months into the internship. - Either way I aim to have the Web Platform ready by March 12. - * Functest VNF implementation phase - This is the last phase that will - involve writing a fully functional implementation of an Open Source VNF - into Functest. I will undertake this after I am 3 months into the - internship. I have a decent familiarity with python and hence I think - it shouldn’t be too difficult. I need to decide how complex the VNFI - should undertake this exercise for (e.g. AAA such as free radius sounds - relatively easy, vCDN is much more challenging). - This will be decided in consent with my mentors. - -Schedule: -========= -I plan to take this project in 6 months time frame as I want to use it -as a chance to read more about NFVs in particular and SDN in general - - -+--------------------------+------------------------------------------+ -| **Date** | **Comment** | -| | | -+--------------------------+------------------------------------------+ -| December 12 - January 12 | Study the above mentioned metrics | -| | Decide which of them are important for | -| | community (and which are not). | -+--------------------------+------------------------------------------+ -| January 12 - January 27 | Make a database for the above studied | -| | metrics and evolve it further based on | -| | Mentors’ input. + associated API | -+--------------------------+------------------------------------------+ -| January 27 - February 5 | Compile the data collected above and make| -| | it public. Although I can keep everything| -| | public from the beginning too. My | -| | rationale of not making the entire data | -| | public in initial stage as the errors | -| | caused by me could be misleading for | -| | developers. | -+--------------------------+------------------------------------------+ -| February 5 - March 5 | Erect the Web Platform and release it | -| | for restricted group for alpha testing. | -+--------------------------+------------------------------------------+ -| March 5 - March 12 | Make it public. Release it to public for | -| | beta testing. Fix Bugs. | -+--------------------------+------------------------------------------+ -| March 12 - April 12 | Start working on implementation of an | -| | Open Source VNF in Functest. | -+--------------------------+------------------------------------------+ -| April 12 - May 12 | I will decided what to do here based on | -| | discussion with mentors. | -+--------------------------+------------------------------------------+ - - -References: -=========== - -.. _`[1]` : Openhub: https://www.openhub.net/explore/projects - -.. _`[2]` : ETSI NFV White Paper: https://portal.etsi.org/Portals/0/TBpages/NFV/Docs/NFV_White_Paper3.pdf - -.. _`[3]` : https://wiki.opnfv.org/display/DEV/Intern+Project%3A+Open+Source+VNF+catalog diff --git a/docs/testing/user/configguide/ci.rst b/docs/testing/user/configguide/ci.rst deleted file mode 100644 index e4fec51dc..000000000 --- a/docs/testing/user/configguide/ci.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-4.0 - -Integration in CI -================= -In CI we use the Docker images and execute the appropriate commands within the -container from Jenkins. - -4 steps have been defined:: - * functest-cleanup: clean existing functest dockers on the jumphost - * functest-daily: run dockers opnfv/functest-* (healthcheck, smoke, features, - vnf) - * functest-store-results: push logs to artifacts - -See `[3]`_ for details. - -.. _`[3]`: https://git.opnfv.org/releng/tree/jjb/functest/functest-daily-jobs.yml diff --git a/docs/testing/user/configguide/conf.py b/docs/testing/user/configguide/conf.py deleted file mode 100644 index c0dc12599..000000000 --- a/docs/testing/user/configguide/conf.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Functest Installation Guide documentation build configuration file, created -# by sphinx-quickstart on Tue Apr 3 03:51:57 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_opnfv_theme - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Functest Installation Guide' -copyright = u'2018, Functest <opnfv-tech-discuss@lists.opnfv.org>' -author = u'Functest <opnfv-tech-discuss@lists.opnfv.org>' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'opnfv' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = { - 'bootswatch_theme': 'journal', - 'navbar_sidebarrel': False, - 'navbar_title': '', -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] -html_theme_path = sphinx_opnfv_theme.get_html_theme_path() - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'FunctestInstallationGuidedoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, - 'FunctestInstallationGuide.tex', - u'Functest Installation Guide Documentation', - u'Functest \\textless{}opnfv-tech-discuss@lists.opnfv.org\\textgreater{}', - 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, - 'functestinstallationguide', - u'Functest Installation Guide Documentation', - [author], - 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, - 'FunctestInstallationGuide', - u'Functest Installation Guide Documentation', - author, - 'FunctestInstallationGuide', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst index c04689191..47c25adb1 100644 --- a/docs/testing/user/configguide/configguide.rst +++ b/docs/testing/user/configguide/configguide.rst @@ -13,31 +13,27 @@ Functest Dockers for OpenStack deployment ----------------------------------------- Docker images are available on the dockerhub: - * opnfv/functest-core * opnfv/functest-healthcheck * opnfv/functest-smoke - * opnfv/functest-features - * opnfv/functest-components + * opnfv/functest-smoke-cntt + * opnfv/functest-benchmarking + * opnfv/functest-benchmarking-cntt * opnfv/functest-vnf -The tag "opnfv-6.0.0" is the official release image in Fraser, but you can -also pull "fraser" tag as it is being maintained by Functest team and might -include bugfixes. - -The Functest docker container environment can -in principle- be also -used with non-OPNFV official installers (e.g. 'devstack'), with the -**disclaimer** that support for such environments is outside of the -scope and responsibility of the OPNFV project. - Preparing your environment ^^^^^^^^^^^^^^^^^^^^^^^^^^ cat env:: - EXTERNAL_NETWORK=XXX DEPLOY_SCENARIO=XXX # if not os-nosdn-nofeature-noha scenario NAMESERVER=XXX # if not 8.8.8.8 + EXTERNAL_NETWORK=XXX # if not first network with router:external=True + DASHBOARD_URL=XXX # else tempest_horizon will be skipped + NEW_USER_ROLE=XXX # if not member + SDN_CONTROLLER_IP=XXX # if odl scenario + VOLUME_DEVICE_NAME=XXX # if not vdb + FLAVOR_EXTRA_SPECS=hw:mem_page_size:large # if fdio scenarios See section on environment variables for details. @@ -50,26 +46,23 @@ cat env_file:: export OS_PROJECT_NAME=XXX export OS_PASSWORD=XXX export OS_IDENTITY_API_VERSION=3 + export OS_REGION_NAME=XXX See section on OpenStack credentials for details. Create a directory for the different images (attached as a Docker volume):: - mkdir -p images && wget -q -O- https://git.opnfv.org/functest/plain/functest/ci/download_images.sh?h=stable/fraser | bash -s -- images && ls -1 images/* - - images/CentOS-7-aarch64-GenericCloud.qcow2 - images/CentOS-7-aarch64-GenericCloud.qcow2.xz - images/CentOS-7-x86_64-GenericCloud.qcow2 - images/cirros-0.4.0-x86_64-disk.img - images/cirros-0.4.0-x86_64-lxc.tar.gz - images/cirros-d161201-aarch64-disk.img - images/cirros-d161201-aarch64-initramfs - images/cirros-d161201-aarch64-kernel - images/cloudify-manager-premium-4.0.1.qcow2 + mkdir -p images && wget -q -O- https://git.opnfv.org/functest/plain/functest/ci/download_images.sh | sh -s -- images && ls -1 images/* + + images/cirros-0.5.1-aarch64-disk.img + images/cirros-0.5.1-x86_64-disk.img + images/cloudify-docker-manager-community-19.01.24.tar + images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2 + images/shaker-image-1.3.0+stretch.qcow2 images/ubuntu-14.04-server-cloudimg-amd64-disk1.img images/ubuntu-14.04-server-cloudimg-arm64-uefi1.img images/ubuntu-16.04-server-cloudimg-amd64-disk1.img - images/vyos-1.1.7.img + images/vyos-1.1.8-amd64.qcow2 Testing healthcheck suite ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -83,14 +76,25 @@ Run healthcheck suite:: Results shall be displayed as follows:: - +----------------------------+------------------+---------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +----------------------------+------------------+---------------------+------------------+----------------+ - | connection_check | functest | healthcheck | 00:07 | PASS | - | api_check | functest | healthcheck | 07:46 | PASS | - | snaps_health_check | functest | healthcheck | 00:36 | PASS | - +----------------------------+------------------+---------------------+------------------+----------------+ - NOTE: the duration is a reference and it might vary depending on your SUT. + +--------------------------+------------------+---------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +--------------------------+------------------+---------------------+------------------+----------------+ + | connection_check | functest | healthcheck | 00:03 | PASS | + | tenantnetwork1 | functest | healthcheck | 00:05 | PASS | + | tenantnetwork2 | functest | healthcheck | 00:06 | PASS | + | vmready1 | functest | healthcheck | 00:06 | PASS | + | vmready2 | functest | healthcheck | 00:08 | PASS | + | singlevm1 | functest | healthcheck | 00:32 | PASS | + | singlevm2 | functest | healthcheck | 00:37 | PASS | + | vping_ssh | functest | healthcheck | 00:46 | PASS | + | vping_userdata | functest | healthcheck | 00:39 | PASS | + | cinder_test | functest | healthcheck | 01:05 | PASS | + | tempest_smoke | functest | healthcheck | 05:39 | PASS | + | tempest_horizon | functest | healthcheck | 01:05 | PASS | + | odl | functest | healthcheck | 00:00 | SKIP | + +--------------------------+------------------+---------------------+------------------+----------------+ + +NOTE: the duration is a reference and it might vary depending on your SUT. Testing smoke suite ^^^^^^^^^^^^^^^^^^^ @@ -104,64 +108,101 @@ Run smoke suite:: Results shall be displayed as follows:: - +------------------------------+------------------+---------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +------------------------------+------------------+---------------+------------------+----------------+ - | vping_ssh | functest | smoke | 00:57 | PASS | - | vping_userdata | functest | smoke | 00:33 | PASS | - | tempest_smoke_serial | functest | smoke | 13:22 | PASS | - | rally_sanity | functest | smoke | 24:07 | PASS | - | refstack_defcore | functest | smoke | 05:21 | PASS | - | patrole | functest | smoke | 04:29 | PASS | - | snaps_smoke | functest | smoke | 46:54 | PASS | - | odl | functest | smoke | 00:00 | SKIP | - | neutron_trunk | functest | smoke | 00:00 | SKIP | - +------------------------------+------------------+---------------+------------------+----------------+ - Note: if the scenario does not support some tests, they are indicated as SKIP. - See User guide for details. - -Testing features suite -^^^^^^^^^^^^^^^^^^^^^^ + +---------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +---------------------------+------------------+---------------+------------------+----------------+ + | tempest_neutron | functest | smoke | 15:30 | PASS | + | tempest_cinder | functest | smoke | 02:01 | PASS | + | tempest_keystone | functest | smoke | 01:17 | PASS | + | tempest_heat | functest | smoke | 22:14 | PASS | + | tempest_telemetry | functest | smoke | 00:00 | SKIP | + | rally_sanity | functest | smoke | 17:24 | PASS | + | refstack_compute | functest | smoke | 07:03 | PASS | + | refstack_object | functest | smoke | 02:09 | PASS | + | refstack_platform | functest | smoke | 07:31 | PASS | + | tempest_full | functest | smoke | 41:52 | PASS | + | tempest_scenario | functest | smoke | 08:42 | PASS | + | tempest_slow | functest | smoke | 43:42 | PASS | + | patrole_admin | functest | smoke | 21:06 | PASS | + | patrole_member | functest | smoke | 21:23 | PASS | + | patrole_reader | functest | smoke | 21:56 | PASS | + | tempest_barbican | functest | smoke | 02:30 | PASS | + | tempest_octavia | functest | smoke | 00:00 | SKIP | + | tempest_cyborg | functest | smoke | 00:00 | SKIP | + +---------------------------+------------------+---------------+------------------+----------------+ + +Note: if the scenario does not support some tests, they are indicated as SKIP. +See User guide for details. + +Testing smoke CNTT suite +^^^^^^^^^^^^^^^^^^^^^^^^ -Run features suite:: +Run smoke-cntt suite:: sudo docker run --env-file env \ -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/images:/home/opnfv/functest/images \ - opnfv/functest-features + opnfv/functest-smoke-cntt Results shall be displayed as follows:: - +-----------------------------+------------------------+------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-----------------------------+------------------------+------------------+------------------+----------------+ - | doctor-notification | doctor | features | 00:00 | SKIP | - | bgpvpn | sdnvpn | features | 00:00 | SKIP | - | functest-odl-sfc | sfc | features | 00:00 | SKIP | - | barometercollectd | barometer | features | 00:00 | SKIP | - | fds | fastdatastacks | features | 00:00 | SKIP | - +-----------------------------+------------------------+------------------+------------------+----------------+ - Note: if the scenario does not support some tests, they are indicated as SKIP. - See User guide for details. - -Testing components suite -^^^^^^^^^^^^^^^^^^^^^^^^ + +-------------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------------------+------------------+---------------+------------------+----------------+ + | tempest_neutron_cntt | functest | smoke | 11:35 | PASS | + | tempest_cinder_cntt | functest | smoke | 01:58 | PASS | + | tempest_keystone_cntt | functest | smoke | 01:13 | PASS | + | tempest_heat_cntt | functest | smoke | 22:32 | PASS | + | rally_sanity_cntt | functest | smoke | 17:16 | PASS | + | tempest_full_cntt | functest | smoke | 41:13 | PASS | + | tempest_scenario_cntt | functest | smoke | 08:57 | PASS | + | tempest_slow_cntt | functest | smoke | 35:58 | PASS | + +-------------------------------+------------------+---------------+------------------+----------------+ + +Testing benchmarking suite +^^^^^^^^^^^^^^^^^^^^^^^^^^ -Run components suite:: +Run benchmarking suite:: sudo docker run --env-file env \ -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/images:/home/opnfv/functest/images \ - opnfv/functest-components + opnfv/functest-benchmarking Results shall be displayed as follows:: - +-------------------------------+------------------+--------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------------------+------------------+--------------------+------------------+----------------+ - | tempest_full_parallel | functest | components | 48:28 | PASS | - | rally_full | functest | components | 126:02 | PASS | - +-------------------------------+------------------+--------------------+------------------+----------------+ + +--------------------+------------------+----------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +--------------------+------------------+----------------------+------------------+----------------+ + | rally_full | functest | benchmarking | 93:03 | PASS | + | rally_jobs | functest | benchmarking | 27:05 | PASS | + | vmtp | functest | benchmarking | 17:56 | PASS | + | shaker | functest | benchmarking | 24:02 | PASS | + +--------------------+------------------+----------------------+------------------+----------------+ + +Note: if the scenario does not support some tests, they are indicated as SKIP. +See User guide for details. + +Testing benchmarking CNTT suite +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Run benchmarking-cntt suite:: + + sudo docker run --env-file env \ + -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \ + -v $(pwd)/images:/home/opnfv/functest/images \ + opnfv/functest-benchmarking-cntt + +Results shall be displayed as follows:: + + +-------------------------+------------------+----------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------------+------------------+----------------------+------------------+----------------+ + | rally_full_cntt | functest | benchmarking | 89:52 | PASS | + | rally_jobs_cntt | functest | benchmarking | 19:39 | PASS | + | vmtp | functest | benchmarking | 16:59 | PASS | + | shaker | functest | benchmarking | 23:43 | PASS | + +-------------------------+------------------+----------------------+------------------+----------------+ Testing vnf suite ^^^^^^^^^^^^^^^^^ @@ -178,34 +219,29 @@ Results shall be displayed as follows:: +----------------------+------------------+--------------+------------------+----------------+ | TEST CASE | PROJECT | TIER | DURATION | RESULT | +----------------------+------------------+--------------+------------------+----------------+ - | cloudify_ims | functest | vnf | 28:15 | PASS | - | vyos_vrouter | functest | vnf | 17:59 | PASS | - | juju_epc | functest | vnf | 46:44 | PASS | + | cloudify | functest | vnf | 05:08 | PASS | + | cloudify_ims | functest | vnf | 24:46 | PASS | + | heat_ims | functest | vnf | 33:12 | PASS | + | vyos_vrouter | functest | vnf | 15:53 | PASS | + | juju_epc | functest | vnf | 27:52 | PASS | +----------------------+------------------+--------------+------------------+----------------+ - Functest Dockers for Kubernetes deployment ------------------------------------------ Docker images are available on the dockerhub: - * opnfv/functest-kubernetes-core - * opnfv/functest-kubernetest-healthcheck - * opnfv/functest-kubernetest-smoke - * opnfv/functest-kubernetest-features + * opnfv/functest-kubernetes-healthcheck + * opnfv/functest-kubernetes-smoke + * opnfv/functest-kubernetes-security + * opnfv/functest-kubernetes-benchmarking + * opnfv/functest-kubernetes-cnf Preparing your environment ^^^^^^^^^^^^^^^^^^^^^^^^^^ cat env:: - INSTALLER_TYPE=XXX - DEPLOY_SCENARIO=XXX - -cat k8s.creds:: - - export KUBERNETES_PROVIDER=local - export KUBE_MASTER_URL=XXX - export KUBE_MASTER_IP=XXX + DEPLOY_SCENARIO=k8s-XXX Testing healthcheck suite ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -213,7 +249,6 @@ Testing healthcheck suite Run healthcheck suite:: sudo docker run -it --env-file env \ - -v $(pwd)/k8s.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/config:/root/.kube/config \ opnfv/functest-kubernetes-healthcheck @@ -222,11 +257,12 @@ volume mapped to ~/.kube/config inside kubernetes container. Results shall be displayed as follows:: - +-------------------------+------------------+-----------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------------+------------------+-----------------+------------------+----------------+ - | k8s_smoke | functest | healthcheck | 01:54 | PASS | - +-------------------------+------------------+-----------------+------------------+----------------+ + +-------------------+------------------+---------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------+------------------+---------------------+------------------+----------------+ + | k8s_quick | functest | healthcheck | 00:18 | PASS | + | k8s_smoke | functest | healthcheck | 01:14 | PASS | + +-------------------+------------------+---------------------+------------------+----------------+ Testing smoke suite ^^^^^^^^^^^^^^^^^^^ @@ -234,45 +270,84 @@ Testing smoke suite Run smoke suite:: sudo docker run -it --env-file env \ - -v $(pwd)/k8s.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/config:/root/.kube/config \ opnfv/functest-kubernetes-smoke Results shall be displayed as follows:: - +-------------------------+------------------+---------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------------+------------------+---------------+------------------+----------------+ - | k8s_conformance | functest | smoke | 57:47 | PASS | - +-------------------------+------------------+---------------+------------------+----------------+ + +---------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +---------------------------+------------------+---------------+------------------+----------------+ + | k8s_conformance | functest | smoke | 94:26 | PASS | + | xrally_kubernetes | functest | smoke | 13:05 | PASS | + +---------------------------+------------------+---------------+------------------+----------------+ -Testing features suite +Testing security suite ^^^^^^^^^^^^^^^^^^^^^^ -Run features suite:: +Run smoke suite:: sudo docker run -it --env-file env \ - -v $(pwd)/k8s.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/config:/root/.kube/config \ - opnfv/functest-kubernetes-features + opnfv/functest-kubernetes-security Results shall be displayed as follows:: - +----------------------+------------------+------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +----------------------+------------------+------------------+------------------+----------------+ - | stor4nfv_k8s | stor4nfv | stor4nfv | 00:00 | SKIP | - | clover_k8s | clover | clover | 00:00 | SKIP | - +----------------------+------------------+------------------+------------------+----------------+ + +---------------------------+------------------+------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +---------------------------+------------------+------------------+------------------+----------------+ + | kube_hunter | functest | security | 00:19 | PASS | + | kube_bench_master | functest | security | 00:02 | PASS | + | kube_bench_node | functest | security | 00:01 | PASS | + +---------------------------+------------------+------------------+------------------+----------------+ + +Testing benchmarking suite +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Run benchmarking suite:: + + sudo docker run -it --env-file env \ + -v $(pwd)/config:/root/.kube/config \ + opnfv/functest-kubernetes-benchmarking + +Results shall be displayed as follows:: + + +--------------------------------+------------------+----------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +--------------------------------+------------------+----------------------+------------------+----------------+ + | xrally_kubernetes_full | functest | benchmarking | 34:16 | PASS | + +--------------------------------+------------------+----------------------+------------------+----------------+ + +Testing cnf suite +^^^^^^^^^^^^^^^^^ + +Run cnf suite:: + + sudo docker run -it --env-file env \ + -v $(pwd)/config:/root/.kube/config \ + opnfv/functest-kubernetes-cnf + +Results shall be displayed as follows:: + + +-------------------------+------------------+--------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------------+------------------+--------------+------------------+----------------+ + | k8s_vims | functest | cnf | 09:06 | PASS | + | helm_vims | functest | cnf | 08:54 | PASS | + | cnf_conformance | functest | cnf | 02:00 | PASS | + +-------------------------+------------------+--------------+------------------+----------------+ Environment variables ===================== -Several environement variables may be specified: - * INSTALLER_TYPE=(apex|compass|daisy|fuel|joid) +Several environment variables may be specified: + * INSTALLER_IP=<Specific IP Address> * DEPLOY_SCENARIO=<vim>-<controller>-<nfv_feature>-<ha_mode> - + * NAMESERVER=XXX # if not 8.8.8.8 + * VOLUME_DEVICE_NAME=XXX # if not vdb + * EXTERNAL_NETWORK=XXX # if not first network with router:external=True + * NEW_USER_ROLE=XXX # if not member INSTALLER_IP is required by Barometer in order to access the installer node and the deployment. @@ -305,8 +380,9 @@ to the Functest Docker Container, using the -e only relevant to Jenkins CI invoked testing and **should not be used** when performing manual test scenarios: - * NODE_NAME = <Test POD Name> - * BUILD_TAG = <Jenkins Build Tag> + * INSTALLER_TYPE=(apex|compass|daisy|fuel) + * NODE_NAME=<Test POD Name> + * BUILD_TAG=<Jenkins Build Tag> where: @@ -389,7 +465,7 @@ By default all the logs are put un /home/opnfv/functest/results/functest.log. If you want to have more logs in console, you may edit the logging.ini file manually. Connect on the docker then edit the file located in -/usr/lib/python2.7/site-packages/xtesting/ci/logging.ini +/usr/lib/python3.8/site-packages/xtesting/ci/logging.ini Change wconsole to console in the desired module to get more traces. @@ -399,8 +475,8 @@ Configuration You may also directly modify the python code or the configuration file (e.g. testcases.yaml used to declare test constraints) under -/usr/lib/python2.7/site-packages/xtesting and -/usr/lib/python2.7/site-packages/functest +/usr/lib/python3.8/site-packages/xtesting and +/usr/lib/python3.8/site-packages/functest Tips @@ -549,7 +625,7 @@ docker container:: Note: In a Jumphost node based on the CentOS family OS, the **nc** commands might not work. You can use the **curl** command instead. - curl http://www.opnfv.org:80 + curl https://www.opnfv.org/ <HTML><HEAD><meta http-equiv="content-type" . @@ -607,11 +683,10 @@ and install the **docker-engine**. The instructions conclude with a work with the above pre-requisite actions. -.. _`[4]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html .. _`dockerdocs`: https://docs.docker.com/ .. _`Proxy`: https://docs.docker.com/engine/admin/systemd/#http-proxy .. _`Install Docker on CentOS`: https://docs.docker.com/engine/installation/linux/centos/ .. _`Functest User Guide`: http://docs.opnfv.org/en/stable-danube/submodules/functest/docs/testing/user/userguide/index.html .. _`images/CentOS-7-x86_64-GenericCloud.qcow2`: https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2 -.. _`images/cirros-0.4.0-x86_64-disk.img`: http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img +.. _`images/cirros-0.5.1-x86_64-disk.img`: http://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img .. _`images/ubuntu-14.04-server-cloudimg-amd64-disk1.img`: https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img diff --git a/docs/testing/user/configguide/index.rst b/docs/testing/user/configguide/index.rst index 5d7eeb6e4..e5963435e 100644 --- a/docs/testing/user/configguide/index.rst +++ b/docs/testing/user/configguide/index.rst @@ -1,38 +1,22 @@ .. SPDX-License-Identifier: CC-BY-4.0 +.. _functest-install-guide: *************************** Functest Installation Guide *************************** .. toctree:: - :numbered: :maxdepth: 2 intro.rst prerequisites.rst configguide.rst - ci.rst References ========== -`[1]`_ : Keystone and public end point constraint - -`[2]`_ : Functest User guide - -`[3]`_ : Functest Jenkins jobs - -`[4]`_ : Functest Configuration guide - -`[5]`_ : OPNFV main site - -`[6]`_ : Functest wiki page +`[1]`_ : Functest Jenkins jobs IRC support channel: #opnfv-functest -.. _`[1]`: https://ask.openstack.org/en/question/68144/keystone-unable-to-use-the-public-endpoint/ -.. _`[2]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/userguide/index.html -.. _`[3]`: https://git.opnfv.org/releng/tree/jjb/functest/functest-daily-jobs.yml -.. _`[4]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html -.. _`[5]`: http://www.opnfv.org -.. _`[6]`: https://wiki.opnfv.org/functest +.. _`[1]`: https://github.com/opnfv/releng/blob/master/jjb/functest/functest.yaml diff --git a/docs/testing/user/configguide/intro.rst b/docs/testing/user/configguide/intro.rst index 72730b253..02329995c 100644 --- a/docs/testing/user/configguide/intro.rst +++ b/docs/testing/user/configguide/intro.rst @@ -64,9 +64,6 @@ follows:: + + + + + + SUT = System Under Test -Note connectivity to management network is not needed for most of the -testcases. But it may be needed for some specific snaps tests. - All the libraries and dependencies needed by all of the Functest tools are pre-installed into the Docker images. This allows running Functest on any platform. @@ -86,6 +83,4 @@ however, it would be possible to adapt them to any OpenStack based VIM + controller environment, since most of the test cases are integrated from upstream communities. -The functional test cases are described in the Functest User `[2]`_ - -.. _`[2]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/userguide/index.html +The functional test cases are described in the :ref:`functest-userguide` diff --git a/docs/testing/user/configguide/prerequisites.rst b/docs/testing/user/configguide/prerequisites.rst index 60be1b93b..dab4a9e6b 100644 --- a/docs/testing/user/configguide/prerequisites.rst +++ b/docs/testing/user/configguide/prerequisites.rst @@ -3,7 +3,7 @@ Prerequisites ============= The OPNFV deployment is out of the scope of this document but it can be -found in http://docs.opnfv.org. +found in https://docs.opnfv.org/en/stable-hunter/. The OPNFV platform is considered as the SUT in this document. Several prerequisites are needed for Functest: @@ -14,13 +14,6 @@ Several prerequisites are needed for Functest: #. An admin/management network created on the SUT #. Connectivity from the Jumphost to the SUT public/external network -Some specific SNAPS tests may require a connectivity from the Jumphost to the -SUT admin/management network but most of the test cases do not. This -requirement can be changed by overriding the 'interface' attribute -(OS_INTERFACE) value to 'public' in the credentials file. Another means to -circumvent this issue would be to change the 'snaps.use_keystone' value from -True to False. - WARNING: Connectivity from Jumphost is essential and it is of paramount importance to make sure it is working before even considering to install and run Functest. Make also sure you understand how your networking is @@ -99,7 +92,4 @@ should thus be known. Ensure you can reach each node in the SUT, from the Jumphost using the 'ping' command using the respective IP address on the public/external network for each node in the SUT. The details of how to determine the needed IP addresses for each node in the SUT may vary according -to the used installer and are therefore ommitted here. - -.. _`[1]`: https://ask.openstack.org/en/question/68144/keystone-unable-to-use-the-public-endpoint/ -.. _`[4]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html +to the used installer and are therefore omitted here. diff --git a/docs/testing/user/userguide/conf.py b/docs/testing/user/userguide/conf.py deleted file mode 100644 index 716d3b3d2..000000000 --- a/docs/testing/user/userguide/conf.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Functest User Guide documentation build configuration file, created by -# sphinx-quickstart on Wed Apr 4 04:05:42 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_opnfv_theme - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Functest User Guide' -copyright = u'2018, Functest <opnfv-tech-discuss@lists.opnfv.org>' -author = u'Functest <opnfv-tech-discuss@lists.opnfv.org>' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'opnfv' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = { - 'bootswatch_theme': 'journal', - 'navbar_sidebarrel': False, - 'navbar_title': '', -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] -html_theme_path = sphinx_opnfv_theme.get_html_theme_path() - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'FunctestUserGuidedoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, - 'FunctestUserGuide.tex', - u'Functest User Guide Documentation', - u'Functest \\textless{}opnfv-tech-discuss@lists.opnfv.org\\textgreater{}', - 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, - 'functestuserguide', - u'Functest User Guide Documentation', - [author], - 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, - 'FunctestUserGuide', - u'Functest User Guide Documentation', - author, - 'FunctestUserGuide', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 8a40ccf4c..1e73cd622 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -12,7 +12,6 @@ Functest User Guide intro.rst test_overview.rst test_details.rst - runfunctest.rst test_results.rst reporting.rst troubleshooting.rst @@ -21,8 +20,6 @@ Functest User Guide References ========== -`[1]`_: Functest configuration guide - `[2]`_: OpenStack Tempest documentation `[3]`_: Rally documentation @@ -31,7 +28,7 @@ References `[5]`_: Clearwater vIMS blueprint -`[6]`_: NIST web site +`[6]`_: Security Content Automation Protocol `[7]`_: OpenSCAP web site @@ -43,9 +40,7 @@ References `[11]`_: Robot Framework web site -`[12]`_: Functest User guide - -`[13]`_: SNAPS wiki +`[13]`_: SNAPS `[14]`_: vRouter @@ -59,25 +54,23 @@ References IRC support chan: #opnfv-functest -.. _`[1]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html -.. _`[2]`: http://docs.openstack.org/developer/tempest/overview.html -.. _`[3]`: https://rally.readthedocs.org/en/latest/index.html -.. _`[4]`: http://events.linuxfoundation.org/sites/events/files/slides/Functest%20in%20Depth_0.pdf +.. _`[2]`: https://docs.openstack.org/tempest/latest/ +.. _`[3]`: https://rally.readthedocs.io/en/latest/index.html +.. _`[4]`: https://events.static.linuxfound.org/sites/events/files/slides/Functest%20in%20Depth_0.pdf .. _`[5]`: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater/blob/master/openstack-blueprint.yaml -.. _`[6]`: https://scap.nist.gov/ +.. _`[6]`: https://en.wikipedia.org/wiki/Security_Content_Automation_Protocol .. _`[7]`: https://github.com/OpenSCAP/openscap .. _`[8]`: https://github.com/openstack/refstack-client -.. _`[9]`: https://github.com/openstack/defcore +.. _`[9]`: https://github.com/openstack/interop .. _`[10]`: https://github.com/openstack/interop/blob/master/2016.08/procedure.rst -.. _`[11]`: http://robotframework.org/ -.. _`[12]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/userguide/index.html -.. _`[13]`: https://wiki.opnfv.org/display/PROJ/SNAPS-OO +.. _`[11]`: https://robotframework.org/ +.. _`[13]`: https://git.opnfv.org/snaps/ .. _`[14]`: https://github.com/oolorg/opnfv-functest-vrouter .. _`[15]`: https://aptira.com/testing-openstack-tempest-part-1/ -.. _`[16]`: http://docs.opnfv.org/en/latest/testing/testing-dev.html -.. _`OPNFV main site`: http://www.opnfv.org -.. _`Functest page`: https://wiki.opnfv.org/functest +.. _`[16]`: http://testresults.opnfv.org/test/ +.. _`OPNFV main site`: https://www.opnfv.org/ +.. _`Functest page`: https://github.com/opnfv/functest/ .. _`OpenRC`: http://docs.openstack.org/user-guide/common/cli_set_environment_variables_using_openstack_rc.html .. _`Rally installation procedure`: https://rally.readthedocs.org/en/latest/tutorial/step_0_installation.html -.. _`config_functest.yaml` : https://git.opnfv.org/cgit/functest/tree/functest/ci/config_functest.yaml +.. _`config_functest.yaml` : https://github.com/opnfv/functest/blob/master/functest/ci/config_functest.yaml .. _`Functest reporting`: http://testresults.opnfv.org/reporting/master/functest/status-apex.html diff --git a/docs/testing/user/userguide/intro.rst b/docs/testing/user/userguide/intro.rst index 0c36cd8c7..c001afb2f 100644 --- a/docs/testing/user/userguide/intro.rst +++ b/docs/testing/user/userguide/intro.rst @@ -7,6 +7,4 @@ The goal of this document is to describe the OPNFV Functest test cases and to provide a procedure to execute them. **IMPORTANT**: It is assumed here that Functest has been properly deployed -following the installation guide procedure `[1]`_. - -.. _`[1]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html +following the installation guide procedure :ref:`functest-install-guide`. diff --git a/docs/testing/user/userguide/reporting.rst b/docs/testing/user/userguide/reporting.rst index 7c8c48ece..8fad55d33 100644 --- a/docs/testing/user/userguide/reporting.rst +++ b/docs/testing/user/userguide/reporting.rst @@ -48,7 +48,7 @@ and features) corresponding to this scenario. All the testcases (X) listed in the table are runnable on os-odl_l2-nofeature scenarios. Please note that other test cases (e.g. sfc_odl, bgpvpn) need ODL configuration -addons and, as a consequence, specific scenario. +add-ons and, as a consequence, specific scenario. There are not considered as runnable on the generic odl_l2 scenario. @@ -82,7 +82,7 @@ Therefore the scoring provides 3 types of indicators: * the maturity: if the percentage (scoring/target scoring * 100) is high, it means that all the tests are PASS * the stability: as the number of iteration is included in the calculation, - the pecentage can be high only if the scenario is run regularly (at least + the percentage can be high only if the scenario is run regularly (at least more than 4 iterations over the last 10 days in CI) In any case, the scoring is used to give feedback to the other projects and diff --git a/docs/testing/user/userguide/runfunctest.rst b/docs/testing/user/userguide/runfunctest.rst deleted file mode 100644 index bb01db7ca..000000000 --- a/docs/testing/user/userguide/runfunctest.rst +++ /dev/null @@ -1,115 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-4.0 - -Executing Functest suites -========================= - -As mentioned in the configuration guide `[1]`_, Alpine docker containers have -been introduced in Euphrates. -Tier containers have been created. -Assuming that you pulled the container and your environement is ready, you can -simply run the tiers by typing (e.g. with functest-healthcheck):: - - sudo docker run --env-file env \ - -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \ - -v $(pwd)/images:/home/opnfv/functest/images \ - opnfv/functest-healthcheck - -You should get:: - - +----------------------------+------------------+---------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +----------------------------+------------------+---------------------+------------------+----------------+ - | connection_check | functest | healthcheck | 00:02 | PASS | - | api_check | functest | healthcheck | 03:19 | PASS | - | snaps_health_check | functest | healthcheck | 00:46 | PASS | - +----------------------------+------------------+---------------------+------------------+----------------+ - -You can run functest-healcheck, functest-smoke, functest-features, -functest-components and functest-vnf. - -The result tables show the results by test case, it can be:: - - * PASS - * FAIL - * SKIP: if the scenario/installer does not support the test case - - -Manual run -========== -If you want to run the test step by step, you may add docker option then run -the different commands within the docker. - -Considering the healthcheck example, running functest manaully means:: - - sudo docker run -ti --env-file env \ - -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \ - -v $(pwd)/images:/home/opnfv/functest/images \ - opnfv/functest-healthcheck /bin/bash - -The docker prompt shall be returned. Then within the docker run the following -commands:: - - $ source /home/opnfv/functest/conf/env_file - -Tier ----- -Each Alpine container provided on the docker hub matches with a tier. -The following commands are available:: - - # functest tier list - - 0. healthcheck: - ['connection_check', 'api_check', 'snaps_health_check'] - # functest tier show healthcheck - +---------------------+---------------+--------------------------+-------------------------------------------------+------------------------------------+ - | TIERS | ORDER | CI LOOP | DESCRIPTION | TESTCASES | - +---------------------+---------------+--------------------------+-------------------------------------------------+------------------------------------+ - | healthcheck | 0 | (daily)|(weekly) | First tier to be executed to verify the | connection_check api_check | - | | | | basic operations in the VIM. | snaps_health_check | - +---------------------+---------------+--------------------------+-------------------------------------------------+------------------------------------+ - -To run all the cases of the tier, type:: - - # functest tier run healthcheck - -Testcase --------- -Testcases can be listed, shown and run though the CLI:: - - # functest testcase list - connection_check - api_check - snaps_health_check - # functest testcase show api_check - +-------------------+--------------------------------------------------+------------------+---------------------------+ - | TEST CASE | DESCRIPTION | CRITERIA | DEPENDENCY | - +-------------------+--------------------------------------------------+------------------+---------------------------+ - | api_check | This test case verifies the retrieval of | 100 | ^((?!lxd).)*$ | - | | OpenStack clients: Keystone, Glance, | | | - | | Neutron and Nova and may perform some | | | - | | simple queries. When the config value of | | | - | | snaps.use_keystone is True, functest | | | - | | must have access to the cloud's private | | | - | | network. | | | - +-------------------+--------------------------------------------------+------------------+---------------------------+ - # functest testcase run connection_check - ... - # functest run all - -You can also type run_tests -t all to run all the tests. - -Note the list of test cases depend on the installer and the scenario. - -Note that the flavors for the SNAPS test cases are able to be configured giving -new metadata values as well as new values for the basic elements of flavor -(i.e. ram, vcpu, disk, ephemeral, swap etc). -The snaps.flavor_extra_specs dict in the config_functest.yaml file could be -used for this purpose. - -Reporting results to the test Database -====================================== -In OPNFV CI we collect all the results from CI. A test API shall be available -as well as a test database `[16]`_. - - -.. _`[1]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html -.. _`[16]`: http://docs.opnfv.org/en/latest/testing/testing-dev.html diff --git a/docs/testing/user/userguide/test_details.rst b/docs/testing/user/userguide/test_details.rst index 992b546f5..98247d488 100644 --- a/docs/testing/user/userguide/test_details.rst +++ b/docs/testing/user/userguide/test_details.rst @@ -24,25 +24,10 @@ The tests are: * *connection_check* - * *api_check* - * *snaps_health_check* -Connection_check consists in 9 test cases (test duration < 5s) checking the +Connection_check consists in test cases (test duration < 5s) checking the connectivity with Glance, Keystone, Neutron, Nova and the external network. -Api_check verifies the retrieval of OpenStack clients: Keystone, Glance, -Neutron and Nova and may perform some simple queries. When the config value of -snaps.use_keystone is True, functest must have access to the cloud's private -network. This suite consists in 49 tests (test duration < 2 minutes). - -Snaps_health_check creates a VM with a single port with an IPv4 address that -is assigned by DHCP and then validates the expected IP with the actual. - -The flavors for the SNAPS test cases are able to be configured giving new -metadata values as well as new values for the basic elements of flavor (i.e. -ram, vcpu, disk, ephemeral, swap etc). The snaps.flavor_extra_specs dict in the -config_functest.yaml file could be used for this purpose. - Self-obviously, successful completion of the 'healthcheck' testcase is a necessary pre-requisite for the execution of all other test Tiers. @@ -228,21 +213,6 @@ NOTE: Test case 'rally_sanity' executes a limited number of Rally smoke test cases. Test case 'rally_full' executes the full defined set of Rally tests. -snaps_smoke ------------- - -This test case contains tests that setup and destroy environments with VMs with -and without Floating IPs with a newly created user and project. Set the config -value snaps.use_floating_ips (True|False) to toggle this functionality. -Please note that When the configuration value of snaps.use_keystone is True, -Functest must have access the cloud's private network. -This suite consists in 120 tests (test duration ~= 50 minutes) - -The flavors for the SNAPS test cases are able to be configured giving new -metadata values as well as new values for the basic elements of flavor (i.e. -ram, vcpu, disk, ephemeral, swap etc). The snaps.flavor_extra_specs dict in -the config_functest.yaml file could be used for this purpose. - SDN Controllers --------------- @@ -302,53 +272,6 @@ Note: the checks in OpenDaylight are based on the returned HTTP status code returned by OpenDaylight. -Features --------- - -Functest has been supporting several feature projects since Brahmaputra: - - -+-----------------+---------+----------+--------+-----------+-----------+ -| Test | Brahma | Colorado | Danube | Euphrates | Fraser | -+=================+=========+==========+========+===========+===========+ -| barometer | | | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| bgpvpn | | X | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| copper | | X | | | | -+-----------------+---------+----------+--------+-----------+-----------+ -| doctor | X | X | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| domino | | X | X | X | | -+-----------------+---------+----------+--------+-----------+-----------+ -| fds | | | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| moon | | X | | | | -+-----------------+---------+----------+--------+-----------+-----------+ -| multisite | | X | X | | | -+-----------------+---------+----------+--------+-----------+-----------+ -| netready | | | X | | | -+-----------------+---------+----------+--------+-----------+-----------+ -| odl_sfc | | X | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| opera | | | X | | | -+-----------------+---------+----------+--------+-----------+-----------+ -| orchestra | | | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| parser | | | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| promise | X | X | X | X | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| security_scan | | X | X | | | -+-----------------+---------+----------+--------+-----------+-----------+ -| clover | | | | | X | -+-----------------+---------+----------+--------+-----------+-----------+ -| stor4nfv | | | | | X | -+-----------------+---------+----------+--------+-----------+-----------+ - -Please refer to the dedicated feature user guides for details. - - VNF --- @@ -376,6 +299,27 @@ The Clearwater architecture is described as follows: :align: center :alt: vIMS architecture +heat_ims +^^^^^^^^ +The IP Multimedia Subsystem or IP Multimedia Core Network Subsystem (IMS) is an +architectural framework for delivering IP multimedia services. + +vIMS has been integrated in Functest to demonstrate the capability to deploy a +relatively complex NFV scenario on the OPNFV platform. The deployment of a +complete functional VNF allows the test of most of the essential functions +needed for a NFV platform. + +The goal of this test suite consists of: + +* deploy a Clearwater vIMS (IP Multimedia Subsystem) VNF using + OpenStack Heat orchestrator based on a HOT template defined in `[17]`_ +* run suite of signaling tests on top of this VNF + +The Clearwater architecture is described as follows: + +.. figure:: ../../../images/clearwater-architecture-v2.png + :align: center + :alt: vIMS architecture vyos-vrouter ^^^^^^^^^^^^ @@ -454,15 +398,15 @@ The kubernetes testcases are distributed across various Tiers: designed to be run with no cloud provider configured. -.. _`[2]`: http://docs.openstack.org/developer/tempest/overview.html -.. _`[3]`: https://rally.readthedocs.org/en/latest/index.html +.. _`[2]`: https://docs.openstack.org/tempest/latest/ +.. _`[3]`: https://rally.readthedocs.io/en/latest/index.html .. _`[5]`: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater/blob/master/openstack-blueprint.yaml .. _`[8]`: https://github.com/openstack/refstack-client -.. _`[9]`: https://github.com/openstack/defcore +.. _`[9]`: https://github.com/openstack/interop .. _`[10]`: https://github.com/openstack/interop/blob/master/2016.08/procedure.rst -.. _`[11]`: http://robotframework.org/ -.. _`[12]`: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/userguide/index.html -.. _`[13]`: https://wiki.opnfv.org/display/PROJ/SNAPS-OO +.. _`[11]`: https://robotframework.org/ +.. _`[13]`: https://git.opnfv.org/snaps/ .. _`[14]`: https://github.com/oolorg/opnfv-functest-vrouter -.. _`[15]`: https://www.rebaca.com/abot-test-orchestration-tool/ -.. _`[16]`: https://github.com/kubernetes/community/blob/master/contributors/devel/e2e-tests.md +.. _`[15]`: https://github.com/RebacaInc/abot_charm +.. _`[16]`: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md +.. _`[17]`: https://github.com/Metaswitch/clearwater-heat/blob/release-129/clearwater.yaml diff --git a/docs/testing/user/userguide/test_overview.rst b/docs/testing/user/userguide/test_overview.rst index 2ff14f732..bc3e79dcb 100644 --- a/docs/testing/user/userguide/test_overview.rst +++ b/docs/testing/user/userguide/test_overview.rst @@ -11,27 +11,19 @@ infrastructure. The current list of test suites can be distributed over 5 main domains: * VIM (Virtualised Infrastructure Manager) * Controllers (i.e. SDN Controllers) - * Features * VNF (Virtual Network Functions) * Kubernetes Functest test suites are also distributed in the OPNFV testing categories: -healthcheck, smoke, features, components, performance, VNF, Stress tests. +healthcheck, smoke, benchmarking, VNF, Stress tests. -All the Healthcheck and smoke tests of a given scenario must be succesful to +All the Healthcheck and smoke tests of a given scenario must be successful to validate the scenario for the release. +-------------+---------------+------------+----------------------------------+ | Domain | Tier | Test case | Comments | +=============+===============+============+==================================+ | VIM | healthcheck | connection | Check OpenStack connectivity | -| | | \_check | through SNAPS framework | -| | +------------+----------------------------------+ -| | | api_check | Check OpenStack API through | -| | | | SNAPS framework | -| | +------------+----------------------------------+ -| | | snaps | basic instance creation, check | -| | | \_health | DHCP | | | | \_check | | | +---------------+------------+----------------------------------+ | | smoke | vping_ssh | NFV "Hello World" using an SSH | @@ -62,9 +54,6 @@ validate the scenario for the release. | | | rally | Run a subset of the OpenStack | | | | \_sanity | Rally Test Suite in smoke mode | | | +------------+----------------------------------+ -| | | snaps\ | Run the SNAPS-OO integration | -| | | \_smoke | tests | -| | +------------+----------------------------------+ | | | refstack | Reference RefStack suite | | | | \_defcore | tempest selection for NFV | | | +------------+----------------------------------+ @@ -98,48 +87,10 @@ validate the scenario for the release. | | | | upstream testcases. See below | | | | | for details | +-------------+---------------+------------+----------------------------------+ -| Features | features | bgpvpn | Implementation of the OpenStack | -| | | | bgpvpn API from the SDNVPN | -| | | | feature project. It allows for | -| | | | the creation of BGP VPNs. | -| | | | See `SDNVPN User Guide`_ for | -| | | | details | -| | +------------+----------------------------------+ -| | | doctor | Doctor platform, as of Colorado | -| | | | release, provides the three | -| | | | features: | -| | | | * Immediate Notification | -| | | | * Consistent resource state | -| | | | awareness for compute host down | -| | | | * Valid compute host status | -| | | | given to VM owner | -| | | | See `Doctor User Guide`_ for | -| | | | details | -| | +------------+----------------------------------+ -| | | odl-sfc | SFC testing for odl scenarios | -| | | | See `SFC User Guide`_ for details| -| | +------------+----------------------------------+ -| | | parser | Parser is an integration project | -| | | | which aims to provide | -| | | | placement/deployment templates | -| | | | translation for OPNFV platform, | -| | | | including TOSCA -> HOT, POLICY ->| -| | | | TOSCA and YANG -> TOSCA. it | -| | | | deals with a fake vRNC. | -| | | | See `Parser User Guide`_ for | -| | | | details | -| | +------------+----------------------------------+ -| | | fds | Test Suite for the OpenDaylight | -| | | | SDN Controller when the GBP | -| | | | features are installed. It | -| | | | integrates some test suites from | -| | | | upstream using Robot as the test | -| | | | framework | -+-------------+---------------+------------+----------------------------------+ | VNF | vnf | cloudify | Example of a real VNF deployment | | | | \_ims | to show the NFV capabilities of | | | | | the platform. The IP Multimedia | -| | | | Subsytem is a typical Telco test | +| | | | Subsystem is a typical Telco test| | | | | case, referenced by ETSI. | | | | | It provides a fully functional | | | | | VoIP System | @@ -164,13 +115,6 @@ validate the scenario for the release. | | smoke | k8s\_ | Run a subset of Kubernetes | | | | conformance| End-to-End tests, expected to | | | | | pass on any Kubernetes cluster | -| +---------------+------------+----------------------------------+ -| | stor4nfv | stor4nfv | Run tests necessary to | -| | | \_k8s | demonstrate conformance of the | -| | | | K8s+Stor4NFV deployment | -| +---------------+------------+----------------------------------+ -| | clover | clover_k8s | Test functionality of | -| | | | K8s+Istio+Clover deployment. | +-------------+---------------+------------+----------------------------------+ @@ -193,7 +137,7 @@ An overview of the Functest Structural Concept is depicted graphically below: Some of the test cases are developed by Functest team members, whereas others are integrated from upstream communities or other OPNFV projects. For example, -`Tempest <http://docs.openstack.org/developer/tempest/overview.html>`_ is the +`Tempest <https://docs.openstack.org/tempest/latest/>`_ is the OpenStack integration test suite and Functest is in charge of the selection, integration and automation of those tests that fit suitably to OPNFV. @@ -229,12 +173,11 @@ combinations (which may change from one version to another): Most of the tests are runnable by any combination, but some tests might have restrictions imposed by the utilized installers or due to the available -deployed features. The system uses the environment variables (INSTALLER_TYPE -and DEPLOY_SCENARIO) to automatically determine the valid test cases, for each -given environment. +deployed services. The system uses the environment variables to automatically +determine the valid test cases, for each given environment. A convenience Functest CLI utility is also available to simplify setting up the -Functest evironment, management of the OpenStack environment (e.g. resource +Functest environment, management of the OpenStack environment (e.g. resource clean-up) and for executing tests. The Functest CLI organised the testcase into logical Tiers, which contain in turn one or more testcases. The CLI allows execution of a single specified @@ -242,10 +185,6 @@ testcase, all test cases in a specified Tier, or the special case of execution of **ALL** testcases. The Functest CLI is introduced in more details in next section. -.. _`[2]`: http://docs.openstack.org/developer/tempest/overview.html -.. _`[3]`: https://rally.readthedocs.org/en/latest/index.html -.. _`Doctor User Guide`: http://artifacts.opnfv.org/doctor/colorado/userguide/index.html -.. _`SDNVPN User Guide`: http://artifacts.opnfv.org/sdnvpn/colorado/docs/userguide/index.html -.. _`Parser User Guide`: http://artifacts.opnfv.org/parser/colorado/docs/userguide/index.html -.. _`Functest Dashboard`: http://testresults.opnfv.org/kibana_dashboards/ -.. _`SFC User Guide`: http://artifacts.opnfv.org/sfc/colorado/userguide/index.html +.. _`[2]`: https://docs.openstack.org/tempest/latest/ +.. _`[3]`: https://rally.readthedocs.io/en/latest/index.html +.. _`Functest Dashboard`: http://testresults.opnfv.org/ diff --git a/docs/testing/user/userguide/test_results.rst b/docs/testing/user/userguide/test_results.rst index bb28989ae..10f87d8ec 100644 --- a/docs/testing/user/userguide/test_results.rst +++ b/docs/testing/user/userguide/test_results.rst @@ -10,7 +10,7 @@ In manual mode test results are displayed in the console and result files are put in /home/opnfv/functest/results. If you want additional logs, you may configure the logging.ini under -/usr/lib/python2.7/site-packages/xtesting/ci. +/usr/lib/python3.8/site-packages/xtesting/ci. Automated testing ----------------- @@ -21,104 +21,143 @@ end of each suite and can be described as follow. Healthcheck suite:: - +----------------------------+------------------+---------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +----------------------------+------------------+---------------------+------------------+----------------+ - | connection_check | functest | healthcheck | 00:07 | PASS | - | api_check | functest | healthcheck | 07:46 | PASS | - | snaps_health_check | functest | healthcheck | 00:36 | PASS | - +----------------------------+------------------+---------------------+------------------+----------------+ + +--------------------------+------------------+---------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +--------------------------+------------------+---------------------+------------------+----------------+ + | connection_check | functest | healthcheck | 00:03 | PASS | + | tenantnetwork1 | functest | healthcheck | 00:05 | PASS | + | tenantnetwork2 | functest | healthcheck | 00:06 | PASS | + | vmready1 | functest | healthcheck | 00:06 | PASS | + | vmready2 | functest | healthcheck | 00:08 | PASS | + | singlevm1 | functest | healthcheck | 00:32 | PASS | + | singlevm2 | functest | healthcheck | 00:37 | PASS | + | vping_ssh | functest | healthcheck | 00:46 | PASS | + | vping_userdata | functest | healthcheck | 00:39 | PASS | + | cinder_test | functest | healthcheck | 01:05 | PASS | + | tempest_smoke | functest | healthcheck | 05:39 | PASS | + | tempest_horizon | functest | healthcheck | 01:05 | PASS | + | odl | functest | healthcheck | 00:00 | SKIP | + +--------------------------+------------------+---------------------+------------------+----------------+ Smoke suite:: - +------------------------------+------------------+---------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +------------------------------+------------------+---------------+------------------+----------------+ - | vping_ssh | functest | smoke | 00:57 | PASS | - | vping_userdata | functest | smoke | 00:33 | PASS | - | tempest_smoke_serial | functest | smoke | 13:22 | PASS | - | rally_sanity | functest | smoke | 24:07 | PASS | - | refstack_defcore | functest | smoke | 05:21 | PASS | - | patrole | functest | smoke | 04:29 | PASS | - | snaps_smoke | functest | smoke | 46:54 | PASS | - | odl | functest | smoke | 00:00 | SKIP | - | neutron_trunk | functest | smoke | 00:00 | SKIP | - +------------------------------+------------------+---------------+------------------+----------------+ - -Features suite:: - - +-----------------------------+------------------------+------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-----------------------------+------------------------+------------------+------------------+----------------+ - | doctor-notification | doctor | features | 00:00 | SKIP | - | bgpvpn | sdnvpn | features | 00:00 | SKIP | - | functest-odl-sfc | sfc | features | 00:00 | SKIP | - | barometercollectd | barometer | features | 00:00 | SKIP | - | fds | fastdatastacks | features | 00:00 | SKIP | - +-----------------------------+------------------------+------------------+------------------+----------------+ - -Components suite:: - - +-------------------------------+------------------+--------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------------------+------------------+--------------------+------------------+----------------+ - | tempest_full_parallel | functest | components | 48:28 | PASS | - | rally_full | functest | components | 126:02 | PASS | - +-------------------------------+------------------+--------------------+------------------+----------------+ + +---------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +---------------------------+------------------+---------------+------------------+----------------+ + | tempest_neutron | functest | smoke | 15:30 | PASS | + | tempest_cinder | functest | smoke | 02:01 | PASS | + | tempest_keystone | functest | smoke | 01:17 | PASS | + | tempest_heat | functest | smoke | 22:14 | PASS | + | tempest_telemetry | functest | smoke | 00:00 | SKIP | + | rally_sanity | functest | smoke | 17:24 | PASS | + | refstack_compute | functest | smoke | 07:03 | PASS | + | refstack_object | functest | smoke | 02:09 | PASS | + | refstack_platform | functest | smoke | 07:31 | PASS | + | tempest_full | functest | smoke | 41:52 | PASS | + | tempest_scenario | functest | smoke | 08:42 | PASS | + | tempest_slow | functest | smoke | 43:42 | PASS | + | patrole_admin | functest | smoke | 21:06 | PASS | + | patrole_member | functest | smoke | 21:23 | PASS | + | patrole_reader | functest | smoke | 21:56 | PASS | + | tempest_barbican | functest | smoke | 02:30 | PASS | + | tempest_octavia | functest | smoke | 00:00 | SKIP | + | tempest_cyborg | functest | smoke | 00:00 | SKIP | + +---------------------------+------------------+---------------+------------------+----------------+ + +Smoke CNTT suite:: + + +-------------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------------------+------------------+---------------+------------------+----------------+ + | tempest_neutron_cntt | functest | smoke | 11:35 | PASS | + | tempest_cinder_cntt | functest | smoke | 01:58 | PASS | + | tempest_keystone_cntt | functest | smoke | 01:13 | PASS | + | tempest_heat_cntt | functest | smoke | 22:32 | PASS | + | rally_sanity_cntt | functest | smoke | 17:16 | PASS | + | tempest_full_cntt | functest | smoke | 41:13 | PASS | + | tempest_scenario_cntt | functest | smoke | 08:57 | PASS | + | tempest_slow_cntt | functest | smoke | 35:58 | PASS | + +-------------------------------+------------------+---------------+------------------+----------------+ + +Benchmarking suite:: + + +--------------------+------------------+----------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +--------------------+------------------+----------------------+------------------+----------------+ + | rally_full | functest | benchmarking | 93:03 | PASS | + | rally_jobs | functest | benchmarking | 27:05 | PASS | + | vmtp | functest | benchmarking | 17:56 | PASS | + | shaker | functest | benchmarking | 24:02 | PASS | + +--------------------+------------------+----------------------+------------------+----------------+ + +Benchmarking CNTT suite:: + + +-------------------------+------------------+----------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------------+------------------+----------------------+------------------+----------------+ + | rally_full_cntt | functest | benchmarking | 89:52 | PASS | + | rally_jobs_cntt | functest | benchmarking | 19:39 | PASS | + | vmtp | functest | benchmarking | 16:59 | PASS | + | shaker | functest | benchmarking | 23:43 | PASS | + +-------------------------+------------------+----------------------+------------------+----------------+ Vnf suite:: - +----------------------+------------------+--------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +----------------------+------------------+--------------+------------------+----------------+ - | cloudify_ims | functest | vnf | 28:15 | PASS | - | vyos_vrouter | functest | vnf | 17:59 | PASS | - | juju_epc | functest | vnf | 46:44 | PASS | - +----------------------+------------------+--------------+------------------+----------------+ - -Parser testcase:: - - +-----------------------+-----------------+------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-----------------------+-----------------+------------------+------------------+----------------+ - | parser-basics | parser | features | 00:00 | SKIP | - +-----------------------+-----------------+------------------+------------------+----------------+ - -Functest Kubernetes test result:: - - +--------------------------------------+------------------------------------------------------------+ - | ENV VAR | VALUE | - +--------------------------------------+------------------------------------------------------------+ - | INSTALLER_TYPE | compass | - | DEPLOY_SCENARIO | k8-nosdn-nofeature-ha | - | BUILD_TAG | jenkins-functest-compass-baremetal-daily-master-75 | - | CI_LOOP | daily | - +--------------------------------------+------------------------------------------------------------+ + +----------------------+------------------+--------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +----------------------+------------------+--------------+------------------+----------------+ + | cloudify | functest | vnf | 05:08 | PASS | + | cloudify_ims | functest | vnf | 24:46 | PASS | + | heat_ims | functest | vnf | 33:12 | PASS | + | vyos_vrouter | functest | vnf | 15:53 | PASS | + | juju_epc | functest | vnf | 27:52 | PASS | + +----------------------+------------------+--------------+------------------+----------------+ Kubernetes healthcheck suite:: - +-------------------+------------------+---------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------+------------------+---------------------+------------------+----------------+ - | k8s_smoke | functest | healthcheck | 01:54 | PASS | - +-------------------+------------------+---------------------+------------------+----------------+ + +-------------------+------------------+---------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------+------------------+---------------------+------------------+----------------+ + | k8s_quick | functest | healthcheck | 00:18 | PASS | + | k8s_smoke | functest | healthcheck | 01:14 | PASS | + +-------------------+------------------+---------------------+------------------+----------------+ Kubernetes smoke suite:: - +-------------------------+------------------+---------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------------+------------------+---------------+------------------+----------------+ - | k8s_conformance | functest | smoke | 57:47 | PASS | - +-------------------------+------------------+---------------+------------------+----------------+ - -Kubernetes features suite:: - - +----------------------+------------------+------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +----------------------+------------------+------------------+------------------+----------------+ - | stor4nfv_k8s | stor4nfv | stor4nfv | 00:00 | SKIP | - | clover_k8s | clover | clover | 00:00 | SKIP | - +----------------------+------------------+------------------+------------------+----------------+ + +---------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +---------------------------+------------------+---------------+------------------+----------------+ + | k8s_conformance | functest | smoke | 94:26 | PASS | + | xrally_kubernetes | functest | smoke | 13:05 | PASS | + +---------------------------+------------------+---------------+------------------+----------------+ + +Kubernetes security suite:: + + +---------------------------+------------------+------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +---------------------------+------------------+------------------+------------------+----------------+ + | kube_hunter | functest | security | 00:19 | PASS | + | kube_bench_master | functest | security | 00:02 | PASS | + | kube_bench_node | functest | security | 00:01 | PASS | + +---------------------------+------------------+------------------+------------------+----------------+ + +Kubernetes benchmarking suite:: + + +--------------------------------+------------------+----------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +--------------------------------+------------------+----------------------+------------------+----------------+ + | xrally_kubernetes_full | functest | benchmarking | 34:16 | PASS | + +--------------------------------+------------------+----------------------+------------------+----------------+ + +Kubernetes cnf suite:: + + +-------------------------+------------------+--------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------------+------------------+--------------+------------------+----------------+ + | k8s_vims | functest | cnf | 09:06 | PASS | + | helm_vims | functest | cnf | 08:54 | PASS | + | cnf_conformance | functest | cnf | 02:00 | PASS | + +-------------------------+------------------+--------------+------------------+----------------+ Results are automatically pushed to the test results database, some additional result files are pushed to OPNFV artifact web sites. diff --git a/docs/testing/user/userguide/troubleshooting.rst b/docs/testing/user/userguide/troubleshooting.rst index 5755544e6..d857ed4c4 100644 --- a/docs/testing/user/userguide/troubleshooting.rst +++ b/docs/testing/user/userguide/troubleshooting.rst @@ -44,8 +44,8 @@ follows:: The Functest CLI is designed to route a call to the corresponding internal python scripts, located in paths:: - /usr/lib/python2.7/site-packages/functest/opnfv_tests/openstack/vping/vping_ssh.py - /usr/lib/python2.7/site-packages/functest/opnfv_tests/openstack/vping/vping_userdata.py + /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/vping/vping_ssh.py + /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/vping/vping_userdata.py Notes: @@ -102,7 +102,7 @@ In this case, proceed to create it manually. These are some hints:: --protocol tcp --port-range-min 80 --port-range-max 80 --remote-ip-prefix 0.0.0.0/0 The next step is to create the instances. The image used is located in -*/home/opnfv/functest/data/cirros-0.4.0-x86_64-disk.img* and a Glance image is +*/home/opnfv/functest/data/cirros-0.5.1-x86_64-disk.img* and a Glance image is created with the name **functest-vping**. If booting the instances fails (i.e. the status is not **ACTIVE**), you can check why it failed by doing:: @@ -125,7 +125,7 @@ This test case creates a floating IP on the external network and assigns it to the second instance **opnfv-vping-2**. The purpose of this is to establish a SSH connection to that instance and SCP a script that will ping the first instance. This script is located in the repository under -/usr/lib/python2.7/site-packages/functest/opnfv_tests/openstack/vping/ping.sh +/usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/vping/ping.sh and takes an IP as a parameter. When the SCP is completed, the test will do a SSH call to that script inside the second instance. Some problems can happen here:: @@ -133,8 +133,8 @@ here:: vPing_ssh- ERROR - Cannot establish connection to IP xxx.xxx.xxx.xxx. Aborting If this is displayed, stop the test or wait for it to finish, if you have used -the special method of test invocation with specific supression of OpenStack -resource clean-up, as explained earler. It means that the Container can not +the special method of test invocation with specific suppression of OpenStack +resource clean-up, as explained earlier. It means that the Container can not reach the Public/External IP assigned to the instance **opnfv-vping-2**. There are many possible reasons, and they really depend on the chosen scenario. For most of the ODL-L3 and ONOS scenarios this has been noticed and it is a known @@ -158,7 +158,7 @@ container:: ping <public IP> If the ping does not return anything, try to ping from the Host where the -Docker container is running. If that solves the problem, check the iptable +Docker container is running. If that solves the problem, check the iptables rules because there might be some rules rejecting ICMP or TCP traffic coming/going from/to the container. @@ -258,42 +258,30 @@ debug information can be found from tempest.log file stored into related Rally deployment folder. Functest offers a possibility to test a customized list of Tempest test cases. -To enable that, add a new entry in docker/components/testcases.yaml on the -"components" container with the following content:: - - - - case_name: tempest_custom - project_name: functest - criteria: 100 - blocking: false - description: >- - The test case allows running a customized list of tempest - test cases - dependencies: - installer: '' - scenario: '' - run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCustom' - -Also, a list of the Tempest test cases must be provided to the container or -modify the existing one in -/usr/lib/python2.7/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/test_list.txt - -Example of custom list of tests 'my-custom-tempest-tests.txt':: - - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke] - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke] +To enable that, add a new entry in docker/smoke/testcases.yaml on the +"smoke" container with the following content:: + + - + case_name: tempest_custom + project_name: functest + criteria: 100 + blocking: false + description: >- + The test case allows running a customized list of tempest + test cases + run: + name: tempest_common + args: + mode: "tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops|\ + tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops" This is an example of running a customized list of Tempest tests in Functest:: sudo docker run --env-file env \ -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/images:/home/opnfv/functest/images \ - -v $(pwd)/my-custom-testcases.yaml:/usr/lib/python2.7/site-packages/functest/ci/testcases.yaml \ - -v $(pwd)/my-custom-tempest-tests.txt:/usr/lib/python2.7/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/test_list.txt \ - opnfv/functest-components run_tests -t tempest_custom - + -v $(pwd)/my-custom-testcases.yaml:/usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml \ + opnfv/functest-smoke run_tests -t tempest_custom Rally ^^^^^ @@ -314,14 +302,14 @@ Possible scenarios are: To know more about what those scenarios are doing, they are defined in directory: -/usr/lib/python2.7/site-packages/functest/opnfv_tests/openstack/rally/scenario +/usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/rally/scenario For more info about Rally scenario definition please refer to the Rally official documentation. `[3]`_ To check any possible problems with Rally, the logs are stored under */home/opnfv/functest/results/rally/* in the Functest Docker container. -.. _`[3]`: https://rally.readthedocs.org/en/latest/index.html +.. _`[3]`: https://rally.readthedocs.io/en/latest/index.html Controllers ----------- |