diff options
32 files changed, 576 insertions, 247 deletions
diff --git a/docs/conf.py b/docs/conf.py index eb12e74b..3c9978bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1 +1,2 @@ from docs_conf.conf import * # noqa: F401,F403 +master_doc = 'index' diff --git a/docs/development/index.rst b/docs/development/index.rst index 2dc16a82..a7d2817b 100644 --- a/docs/development/index.rst +++ b/docs/development/index.rst @@ -2,18 +2,18 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) 2016 OPNFV. +.. _development: -====== -Doctor -====== +=========== +Development +=========== .. toctree:: :maxdepth: 2 - ./design/index.rst - ./requirements/index.rst - ./manuals/index.rst - ./overview/functest_scenario/index.rst + ./design/index + ./overview/index + ./requirements/index Indices ======= diff --git a/docs/development/overview/index.rst b/docs/development/overview/index.rst index 956e73e3..f6d78d57 100644 --- a/docs/development/overview/index.rst +++ b/docs/development/overview/index.rst @@ -3,11 +3,12 @@ .. _doctor-overview: -************************ -Doctor Development Guide -************************ +******** +Overview +******** .. toctree:: :maxdepth: 2 + overview.rst testing.rst diff --git a/docs/development/overview/overview.rst b/docs/development/overview/overview.rst new file mode 100644 index 00000000..21f5439e --- /dev/null +++ b/docs/development/overview/overview.rst @@ -0,0 +1,52 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Platform overview +""""""""""""""""" + +Doctor platform provides these features since `Danube Release <https://wiki.opnfv.org/display/SWREL/Danube>`_: + +* Immediate Notification +* Consistent resource state awareness for compute host down +* Valid compute host status given to VM owner + +These features enable high availability of Network Services on top of +the virtualized infrastructure. Immediate notification allows VNF managers +(VNFM) to process recovery actions promptly once a failure has occurred. +Same framework can also be utilized to have VNFM awareness about +infrastructure maintenance. + +Consistency of resource state is necessary to execute recovery actions +properly in the VIM. + +Ability to query host status gives VM owner the possibility to get +consistent state information through an API in case of a compute host +fault. + +The Doctor platform consists of the following components: + +* OpenStack Compute (Nova) +* OpenStack Networking (Neutron) +* OpenStack Telemetry (Ceilometer) +* OpenStack Alarming (AODH) +* Doctor Sample Inspector, OpenStack Congress or OpenStack Vitrage +* Doctor Sample Monitor or any monitor supported by Congress or Vitrage + +.. note:: + Doctor Sample Monitor is used in Doctor testing. However in real + implementation like Vitrage, there are several other monitors supported. + +You can see an overview of the Doctor platform and how components interact in +:numref:`figure-p1`. + + +Maintenance use case provides these features since `Iruya Release <https://wiki.opnfv.org/display/SWREL/Iruya>`_: + +* Infrastructure maintenance and upgrade workflow +* Interaction between VNFM and infrastructe workflow + +Since `Jerma Release <https://wiki.opnfv.org/display/SWREL/Jerma>`_ maintenance +use case also supports 'ETSI FEAT03' implementation to have the infrastructure +maintenance and upgrade fully optimized while keeping zero impact on VNF +service. + diff --git a/docs/development/requirements/index.rst b/docs/development/requirements/index.rst index fceaebf0..ccc35cb8 100644 --- a/docs/development/requirements/index.rst +++ b/docs/development/requirements/index.rst @@ -3,9 +3,9 @@ .. _doctor-requirements: -**************************************** -Doctor: Fault Management and Maintenance -**************************************** +********************************************** +Requirements: Fault Management and Maintenance +********************************************** :Project: Doctor, https://wiki.opnfv.org/doctor :Editors: Ashiq Khan (NTT DOCOMO), Gerald Kunzmann (NTT DOCOMO) diff --git a/docs/index.rst b/docs/index.rst index 4dedb98d..b8e8bfd0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,6 @@ Fault Management and Maintenance (Doctor) :numbered: :maxdepth: 2 - release/index development/index - + release/index + testing/index diff --git a/docs/release/configguide/feature.configuration.rst b/docs/release/configguide/feature.configuration.rst index 64928eea..8fbff50e 100644 --- a/docs/release/configguide/feature.configuration.rst +++ b/docs/release/configguide/feature.configuration.rst @@ -159,3 +159,57 @@ You can configure the Sample Monitor as follows (Example for Apex deployment): "http://127.0.0.1:$INSPECTOR_PORT/events" > monitor.log 2>&1 & **Collectd Monitor** + +OpenStack components +==================== + +In OPNFV and with Doctor testing you can have all OpenStack components configured +as needed. Here is sample of the needed configuration modifications. + +Ceilometer +---------- + +/etc/ceilometer/event_definitions.yaml: +# Maintenance use case needs new alarm definitions to be added +- event_type: maintenance.scheduled + traits: + actions_at: + fields: payload.maintenance_at + type: datetime + allowed_actions: + fields: payload.allowed_actions + host_id: + fields: payload.host_id + instances: + fields: payload.instances + metadata: + fields: payload.metadata + project_id: + fields: payload.project_id + reply_url: + fields: payload.reply_url + session_id: + fields: payload.session_id + state: + fields: payload.state +- event_type: maintenance.host + traits: + host: + fields: payload.host + project_id: + fields: payload.project_id + session_id: + fields: payload.session_id + state: + fields: payload.state + +/etc/ceilometer/event_pipeline.yaml: +# Maintenance and Fault management both needs these to be added + - notifier:// + - notifier://?topic=alarm.all + +Nova +---- + +/etc/nova/nova.conf +cpu_allocation_ratio=1.0 diff --git a/docs/release/configguide/index.rst b/docs/release/configguide/index.rst index b1e7c33d..c2331115 100644 --- a/docs/release/configguide/index.rst +++ b/docs/release/configguide/index.rst @@ -3,9 +3,9 @@ .. _doctor-configguide: -************************* -Doctor Installation Guide -************************* +************************** +Doctor Configuration Guide +************************** .. toctree:: :maxdepth: 2 diff --git a/docs/release/index.rst b/docs/release/index.rst index 8a1bf405..67eb4c5f 100644 --- a/docs/release/index.rst +++ b/docs/release/index.rst @@ -2,14 +2,18 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) 2017 OPNFV. +.. _release: -====== -Doctor -====== +======= +Release +======= .. toctree:: :maxdepth: 2 + ./configguide/index.rst ./installation/index.rst + ./release-notes/index.rst + ./scenarios/fault_management/fault_management.rst + ./scenarios/maintenance/maintenance.rst ./userguide/index.rst - diff --git a/docs/development/manuals/index.rst b/docs/release/installation/index.rst index f705f94a..f6527e5d 100644 --- a/docs/development/manuals/index.rst +++ b/docs/release/installation/index.rst @@ -1,13 +1,13 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. _doctor-manuals: +.. _doctor-configguide: -******* -Manuals -******* +************************* +Doctor Installation Guide +************************* .. toctree:: + :maxdepth: 2 -.. include:: mark-host-down_manual.rst -.. include:: get-valid-server-state.rst + installation.rst diff --git a/docs/release/installation/installation.rst b/docs/release/installation/installation.rst new file mode 100644 index 00000000..564f19fd --- /dev/null +++ b/docs/release/installation/installation.rst @@ -0,0 +1,44 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Doctor Installation +==================== + +You can clone doctor project in OPNFV installer jumphost or if you are not +in OPNFV environment you can clone Doctor to DevStack controller node + +git clone https://gerrit.opnfv.org/gerrit/doctor + +In DevStack controller here is a sample of including what Doctor testing +will require for sample fault management testing and for maintenance +testing using Fenix + +.. code-block:: bash + + git clone https://github.com/openstack/devstack -b stable/train + +.. code-block:: bash + + cd devstack vi local.conf + +.. code-block:: bash + + [[local|localrc]] + GIT_BASE=https://git.openstack.org + HOST_IP=<host_ip> + ADMIN_PASSWORD=admin + DATABASE_PASSWORD=admin + RABBIT_PASSWORD=admin + SERVICE_PASSWORD=admin + LOGFILE=/opt/stack/stack.sh.log + + PUBLIC_INTERFACE=eth0 + + CEILOMETER_EVENT_ALARM=True + + ENABLED_SERVICES=key,rabbit,mysql,fenix-engine,fenix-api,aodh-evaluator,aodh-notifier,aodh-api + + enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer stable/train + enable_plugin aodh https://git.openstack.org/openstack/aodh stable/train + enable_plugin gnocchi https://github.com/openstack/gnocchi + enable_plugin fenix https://opendev.org/x/fenix master diff --git a/docs/release/release-notes/release-notes.rst b/docs/release/release-notes/release-notes.rst index 92775557..b525335e 100644 --- a/docs/release/release-notes/release-notes.rst +++ b/docs/release/release-notes/release-notes.rst @@ -7,33 +7,41 @@ This document provides the release notes for Iruya version of Doctor. Important notes =============== -In Iruya release there has not been many changes. - -All testing is now being made with Fuel installer. Maintenance use case -is now only tested against latest upstream Fenix. Only sample inspector is -tested as Fuel do not support Vitrage or Congress. +Jerma release has mainly been for finalizing maintenance use case testing +supporting the ETSI FEAT03 defined interactino between VNFM and infrastructure. +This is mainly to have infrastructure maintenance and upgrade operations +opttimized as fast as they can while keeping VNFs on top with zero impact +on their service. + +Further more this is the final release of Doctor and the more deep testing is +moving more to upstream projects like Fenix for the maintenance. Also in +this release we have made sure that all Doctor testing and any deeper testing +with ehe upstream projects can be done in DevStack. This also makes DevStack +the most important installer. Summary ======= -Iruya Doctor framework uses OpenStack Stein integrated into its test cases. +Jerma Doctor framework uses OpenStack Train integrated into its test cases. Release Data ============ Doctor changes -- Maintenance use case updated to support latest version of Fenix running - in container on controller node -- Maintenance use case now support Fuel installer -- Doctor updated to use OpenStack Stein and only python 3.6 -- Testing only sample inspector as lacking installer support for - Vitrage and Congress +- Maintenance use case updated to support latest version of Fenix. +- Maintenance use case now supports ETSI FEAT03 optimization with Fenix. +- Doctor testing is now preferred to be done in DevStack environment + where one can easily select OpenStack release from Rocky to Ussuri to + test Doctor functionality. Latest OPNFV Fuel can also be used for the + OpenStack version it supports. -Releng changes +Doctor CI -- Doctor testing running with python 3.6 and with sample inspector -- Doctor is only tested with Fuel installer +- Doctor tested with fuel installer. +- Fault management use case is tested with sample inspector. +- Maintenance use case is tested with sample implementation and towards + the latest Fenix version. The includes the new ETSI FEAT03 optimization. Version change ^^^^^^^^^^^^^^ @@ -41,12 +49,13 @@ Version change Module version changes ~~~~~~~~~~~~~~~~~~~~~~ -- OpenStack has changed from Rocky to Stein since previous Hunter release. +- OpenStack has changed Train Document version changes ~~~~~~~~~~~~~~~~~~~~~~~~ -N/A +All documentation is updated to OPNFV unified format according to +documentation guidelines. Small updates in many documents. Reason for version ^^^^^^^^^^^^^^^^^^ @@ -56,11 +65,14 @@ N/A Feature additions ~~~~~~~~~~~~~~~~~ -+--------------------+--------------------------------------------------------------+ -| **JIRA REFERENCE** | **SLOGAN** | -+--------------------+--------------------------------------------------------------+ -| DOCTOR-134 | Update Doctor maintenance use case to work with latest Fenix | -+--------------------+--------------------------------------------------------------+ ++--------------------+--------------------------------------------+ +| **JIRA REFERENCE** | **SLOGAN** | ++--------------------+--------------------------------------------+ +| DOCTOR-137 | VNFM maintenance with ETSI changes | ++--------------------+--------------------------------------------+ +| DOCTOR-136 | DevStack support | ++--------------------+--------------------------------------------+ + Deliverables ------------ @@ -127,3 +139,8 @@ References For more information about the OPNFV Doctor latest work, please see: https://wiki.opnfv.org/display/doctor/Doctor+Home + +Further information about ETSI FEAT03 optimization can be found from Fenix +Documentation: + +https://fenix.readthedocs.io/en/latest diff --git a/docs/release/release-notes/releasenotes_iruya.rst b/docs/release/release-notes/releasenotes_iruya.rst new file mode 100644 index 00000000..92775557 --- /dev/null +++ b/docs/release/release-notes/releasenotes_iruya.rst @@ -0,0 +1,129 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + + +This document provides the release notes for Iruya version of Doctor. + +Important notes +=============== + +In Iruya release there has not been many changes. + +All testing is now being made with Fuel installer. Maintenance use case +is now only tested against latest upstream Fenix. Only sample inspector is +tested as Fuel do not support Vitrage or Congress. + +Summary +======= + +Iruya Doctor framework uses OpenStack Stein integrated into its test cases. + +Release Data +============ + +Doctor changes + +- Maintenance use case updated to support latest version of Fenix running + in container on controller node +- Maintenance use case now support Fuel installer +- Doctor updated to use OpenStack Stein and only python 3.6 +- Testing only sample inspector as lacking installer support for + Vitrage and Congress + +Releng changes + +- Doctor testing running with python 3.6 and with sample inspector +- Doctor is only tested with Fuel installer + +Version change +^^^^^^^^^^^^^^ + +Module version changes +~~~~~~~~~~~~~~~~~~~~~~ + +- OpenStack has changed from Rocky to Stein since previous Hunter release. + +Document version changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +N/A + +Reason for version +^^^^^^^^^^^^^^^^^^ + +N/A + +Feature additions +~~~~~~~~~~~~~~~~~ + ++--------------------+--------------------------------------------------------------+ +| **JIRA REFERENCE** | **SLOGAN** | ++--------------------+--------------------------------------------------------------+ +| DOCTOR-134 | Update Doctor maintenance use case to work with latest Fenix | ++--------------------+--------------------------------------------------------------+ + +Deliverables +------------ + +Software deliverables +===================== + +None + +Documentation deliverables +========================== + +https://git.opnfv.org/doctor/tree/docs + +Known Limitations, Issues and Workarounds +========================================= + +System Limitations +^^^^^^^^^^^^^^^^^^ + +Maintenance test case requirements: + +- Minimum number of nodes: 1 Controller, 3 Computes +- Min number of VCPUs: 2 VCPUs for each compute + +Known issues +^^^^^^^^^^^^ + +None + +Workarounds +^^^^^^^^^^^ + +None + +Test Result +=========== + +Doctor CI results with TEST_CASE='fault_management' and INSPECTOR_TYPE=sample +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++--------------------------------------+--------------+ +| **TEST-SUITE** | **Results:** | ++--------------------------------------+--------------+ +| INSTALLER_TYPE='fuel' | SUCCESS | ++--------------------------------------+--------------+ + +Doctor CI results with TEST_CASE='maintenance' and INSPECTOR_TYPE=sample +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++--------------------------------------+--------------+ +| **TEST-SUITE** | **Results:** | ++--------------------------------------+--------------+ +| INSTALLER_TYPE='fuel' | SUCCESS | +| ADMIN_TOOL_TYPE='fenix' *) | | ++--------------------------------------+--------------+ + +*) Sample implementation not updated according to latest upstream Fenix + and is currently not being tested. + +References +========== + +For more information about the OPNFV Doctor latest work, please see: + +https://wiki.opnfv.org/display/doctor/Doctor+Home diff --git a/docs/release/scenarios/fault_management/fault_management.rst b/docs/release/scenarios/fault_management/fault_management.rst new file mode 100644 index 00000000..99371201 --- /dev/null +++ b/docs/release/scenarios/fault_management/fault_management.rst @@ -0,0 +1,90 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + + +Running test cases +"""""""""""""""""" + +Functest will call the "doctor_tests/main.py" in Doctor to run the test job. +Doctor testing can also be triggered by tox on OPNFV installer jumphost. Tox +is normally used for functional, module and coding style testing in Python +project. + +Currently 'MCP' and 'devstack' installer are supported. + + +Fault management use case +""""""""""""""""""""""""" + +* A consumer of the NFVI wants to receive immediate notifications about faults + in the NFVI affecting the proper functioning of the virtual resources. + Therefore, such faults have to be detected as quickly as possible, and, when + a critical error is observed, the affected consumer is immediately informed + about the fault and can switch over to the STBY configuration. + +The faults to be monitored (and at which detection rate) will be configured by +the consumer. Once a fault is detected, the Inspector in the Doctor +architecture will check the resource map maintained by the Controller, to find +out which virtual resources are affected and then update the resources state. +The Notifier will receive the failure event requests sent from the Controller, +and notify the consumer(s) of the affected resources according to the alarm +configuration. + +Detailed workflow information is as follows: + +* Consumer(VNFM): (step 0) creates resources (network, server/instance) and an + event alarm on state down notification of that server/instance or Neutron + port. + +* Monitor: (step 1) periodically checks nodes, such as ping from/to each + dplane nic to/from gw of node, (step 2) once it fails to send out event + with "raw" fault event information to Inspector + +* Inspector: when it receives an event, it will (step 3) mark the host down + ("mark-host-down"), (step 4) map the PM to VM, and change the VM status to + down. In network failure case, also Neutron port is changed to down. + +* Controller: (step 5) sends out instance update event to Ceilometer. In network + failure case, also Neutron port is changed to down and corresponding event is + sent to Ceilometer. + +* Notifier: (step 6) Ceilometer transforms and passes the events to AODH, + (step 7) AODH will evaluate events with the registered alarm definitions, + then (step 8) it will fire the alarm to the "consumer" who owns the + instance + +* Consumer(VNFM): (step 9) receives the event and (step 10) recreates a new + instance + +Fault management test case +"""""""""""""""""""""""""" + +Functest will call the 'doctor-test' command in Doctor to run the test job. + +The following steps are executed: + +Firstly, get the installer ip according to the installer type. Then ssh to +the installer node to get the private key for accessing to the cloud. As +'fuel' installer, ssh to the controller node to modify nova and ceilometer +configurations. + +Secondly, prepare image for booting VM, then create a test project and test +user (both default to doctor) for the Doctor tests. + +Thirdly, boot a VM under the doctor project and check the VM status to verify +that the VM is launched completely. Then get the compute host info where the VM +is launched to verify connectivity to the target compute host. Get the consumer +ip according to the route to compute ip and create an alarm event in Ceilometer +using the consumer ip. + +Fourthly, the Doctor components are started, and, based on the above preparation, +a failure is injected to the system, i.e. the network of compute host is +disabled for 3 minutes. To ensure the host is down, the status of the host +will be checked. + +Finally, the notification time, i.e. the time between the execution of step 2 +(Monitor detects failure) and step 9 (Consumer receives failure notification) +is calculated. + +According to the Doctor requirements, the Doctor test is successful if the +notification time is below 1 second. diff --git a/docs/development/overview/functest_scenario/images/Fault-management-design.png b/docs/release/scenarios/maintenance/images/Fault-management-design.png Binary files differindex 6d98cdec..6d98cdec 100644 --- a/docs/development/overview/functest_scenario/images/Fault-management-design.png +++ b/docs/release/scenarios/maintenance/images/Fault-management-design.png diff --git a/docs/development/overview/functest_scenario/images/LICENSE b/docs/release/scenarios/maintenance/images/LICENSE index 21a2d03d..21a2d03d 100644 --- a/docs/development/overview/functest_scenario/images/LICENSE +++ b/docs/release/scenarios/maintenance/images/LICENSE diff --git a/docs/development/overview/functest_scenario/images/Maintenance-design.png b/docs/release/scenarios/maintenance/images/Maintenance-design.png Binary files differindex 8f21db6a..8f21db6a 100644 --- a/docs/development/overview/functest_scenario/images/Maintenance-design.png +++ b/docs/release/scenarios/maintenance/images/Maintenance-design.png diff --git a/docs/development/overview/functest_scenario/images/Maintenance-workflow.png b/docs/release/scenarios/maintenance/images/Maintenance-workflow.png Binary files differindex 9b65fd59..9b65fd59 100644 --- a/docs/development/overview/functest_scenario/images/Maintenance-workflow.png +++ b/docs/release/scenarios/maintenance/images/Maintenance-workflow.png diff --git a/docs/development/overview/functest_scenario/doctor-scenario-in-functest.rst b/docs/release/scenarios/maintenance/maintenance.rst index 4505dd8f..ecfe76b1 100644 --- a/docs/development/overview/functest_scenario/doctor-scenario-in-functest.rst +++ b/docs/release/scenarios/maintenance/maintenance.rst @@ -2,142 +2,6 @@ .. http://creativecommons.org/licenses/by/4.0 - -Platform overview -""""""""""""""""" - -Doctor platform provides these features since `Danube Release <https://wiki.opnfv.org/display/SWREL/Danube>`_: - -* Immediate Notification -* Consistent resource state awareness for compute host down -* Valid compute host status given to VM owner - -These features enable high availability of Network Services on top of -the virtualized infrastructure. Immediate notification allows VNF managers -(VNFM) to process recovery actions promptly once a failure has occurred. -Same framework can also be utilized to have VNFM awareness about -infrastructure maintenance. - -Consistency of resource state is necessary to execute recovery actions -properly in the VIM. - -Ability to query host status gives VM owner the possibility to get -consistent state information through an API in case of a compute host -fault. - -The Doctor platform consists of the following components: - -* OpenStack Compute (Nova) -* OpenStack Networking (Neutron) -* OpenStack Telemetry (Ceilometer) -* OpenStack Alarming (AODH) -* Doctor Sample Inspector, OpenStack Congress or OpenStack Vitrage -* Doctor Sample Monitor or any monitor supported by Congress or Vitrage - -.. note:: - Doctor Sample Monitor is used in Doctor testing. However in real - implementation like Vitrage, there are several other monitors supported. - -You can see an overview of the Doctor platform and how components interact in -:numref:`figure-p1`. - -.. figure:: ./images/Fault-management-design.png - :name: figure-p1 - :width: 100% - - Doctor platform and typical sequence - -Detailed information on the Doctor architecture can be found in the Doctor -requirements documentation: -http://artifacts.opnfv.org/doctor/docs/requirements/05-implementation.html - -Running test cases -"""""""""""""""""" - -Functest will call the "doctor_tests/main.py" in Doctor to run the test job. -Doctor testing can also be triggered by tox on OPNFV installer jumphost. Tox -is normally used for functional, module and coding style testing in Python -project. - -Currently, 'Apex', 'MCP' and 'local' installer are supported. - - -Fault management use case -""""""""""""""""""""""""" - -* A consumer of the NFVI wants to receive immediate notifications about faults - in the NFVI affecting the proper functioning of the virtual resources. - Therefore, such faults have to be detected as quickly as possible, and, when - a critical error is observed, the affected consumer is immediately informed - about the fault and can switch over to the STBY configuration. - -The faults to be monitored (and at which detection rate) will be configured by -the consumer. Once a fault is detected, the Inspector in the Doctor -architecture will check the resource map maintained by the Controller, to find -out which virtual resources are affected and then update the resources state. -The Notifier will receive the failure event requests sent from the Controller, -and notify the consumer(s) of the affected resources according to the alarm -configuration. - -Detailed workflow information is as follows: - -* Consumer(VNFM): (step 0) creates resources (network, server/instance) and an - event alarm on state down notification of that server/instance or Neutron - port. - -* Monitor: (step 1) periodically checks nodes, such as ping from/to each - dplane nic to/from gw of node, (step 2) once it fails to send out event - with "raw" fault event information to Inspector - -* Inspector: when it receives an event, it will (step 3) mark the host down - ("mark-host-down"), (step 4) map the PM to VM, and change the VM status to - down. In network failure case, also Neutron port is changed to down. - -* Controller: (step 5) sends out instance update event to Ceilometer. In network - failure case, also Neutron port is changed to down and corresponding event is - sent to Ceilometer. - -* Notifier: (step 6) Ceilometer transforms and passes the events to AODH, - (step 7) AODH will evaluate events with the registered alarm definitions, - then (step 8) it will fire the alarm to the "consumer" who owns the - instance - -* Consumer(VNFM): (step 9) receives the event and (step 10) recreates a new - instance - -Fault management test case -"""""""""""""""""""""""""" - -Functest will call the 'doctor-test' command in Doctor to run the test job. - -The following steps are executed: - -Firstly, get the installer ip according to the installer type. Then ssh to -the installer node to get the private key for accessing to the cloud. As -'fuel' installer, ssh to the controller node to modify nova and ceilometer -configurations. - -Secondly, prepare image for booting VM, then create a test project and test -user (both default to doctor) for the Doctor tests. - -Thirdly, boot a VM under the doctor project and check the VM status to verify -that the VM is launched completely. Then get the compute host info where the VM -is launched to verify connectivity to the target compute host. Get the consumer -ip according to the route to compute ip and create an alarm event in Ceilometer -using the consumer ip. - -Fourthly, the Doctor components are started, and, based on the above preparation, -a failure is injected to the system, i.e. the network of compute host is -disabled for 3 minutes. To ensure the host is down, the status of the host -will be checked. - -Finally, the notification time, i.e. the time between the execution of step 2 -(Monitor detects failure) and step 9 (Consumer receives failure notification) -is calculated. - -According to the Doctor requirements, the Doctor test is successful if the -notification time is below 1 second. - Maintenance use case """""""""""""""""""" @@ -249,7 +113,8 @@ After all computes are maintained, `admin tool` can send `MAINTENANCE_COMPLETE` to tell maintenance/upgrade is now complete. For `app manager` this means he can scale back to full capacity. -This is the current sample implementation and test case. Real life -implementation is started in OpenStack Fenix project and there we should -eventually address requirements more deeply and update the test case with Fenix -implementation. +There is currently sample implementation on VNFM and test case. In +infrastructure side there is sample implementation of 'admin_tool' and +there is also support for the OpenStack Fenix that extends the use case to +support 'ETSI FEAT03' for VNFM interaction and to optimize the whole +infrastructure mainteannce and upgrade. diff --git a/docs/development/manuals/get-valid-server-state.rst b/docs/release/userguide/get-valid-server-state.rst index 824ea3c2..824ea3c2 100644 --- a/docs/development/manuals/get-valid-server-state.rst +++ b/docs/release/userguide/get-valid-server-state.rst diff --git a/docs/release/userguide/index.rst b/docs/release/userguide/index.rst index eee855dc..577072c7 100644 --- a/docs/release/userguide/index.rst +++ b/docs/release/userguide/index.rst @@ -11,3 +11,6 @@ Doctor User Guide :maxdepth: 2 feature.userguide.rst + get-valid-server-state.rst + mark-host-down_manual.rst + monitors.rst diff --git a/docs/development/manuals/mark-host-down_manual.rst b/docs/release/userguide/mark-host-down_manual.rst index 3815205d..3815205d 100644 --- a/docs/development/manuals/mark-host-down_manual.rst +++ b/docs/release/userguide/mark-host-down_manual.rst diff --git a/docs/development/manuals/monitors.rst b/docs/release/userguide/monitors.rst index eeb5e226..eeb5e226 100644 --- a/docs/development/manuals/monitors.rst +++ b/docs/release/userguide/monitors.rst diff --git a/docs/testing/developer/index.rst b/docs/testing/developer/index.rst new file mode 100644 index 00000000..dfbcfa74 --- /dev/null +++ b/docs/testing/developer/index.rst @@ -0,0 +1,13 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +********* +Developer +********* + +.. toctree:: + :numbered: + :maxdepth: 2 + + testing.rst diff --git a/docs/development/overview/testing.rst b/docs/testing/developer/testing.rst index 663d4c3f..6a929130 100644 --- a/docs/development/overview/testing.rst +++ b/docs/testing/developer/testing.rst @@ -38,11 +38,19 @@ export TEST_CASE with different values: export TEST_CASE='fault_management' #Maintenance (requires 3 compute nodes) export TEST_CASE='maintenance' - #Use Fenix in maintenance testing instead of sample admin_tool - export ADMIN_TOOL_TYPE='fenix' #Run both tests cases export TEST_CASE='all' + #Use Fenix in maintenance testing instead of sample admin_tool + #This is only for 'mainteanance' test case + export ADMIN_TOOL_TYPE='fenix' + export APP_MANAGER_TYPE='vnfm' + + #Run in different installer jumphost 'fuel' or 'apex' + #In multinode DevStack you run Doctor in controller node + #with value export APP_MANAGER_TYPE=vnfm + export INSTALLER_TYPE='fuel' + Run Python Test Script ~~~~~~~~~~~~~~~~~~~~~~ @@ -59,7 +67,8 @@ environment and then run the test. .. _doctor.sample.conf: https://git.opnfv.org/doctor/tree/etc/doctor.sample.conf -In OPNFV Apex jumphost you can run Doctor testing as follows using tox: +In OPNFV testing environment jumphost you can run Doctor testing as follows +using tox: .. code-block:: bash @@ -69,31 +78,5 @@ In OPNFV Apex jumphost you can run Doctor testing as follows using tox: git clone https://gerrit.opnfv.org/gerrit/doctor cd doctor sudo -E tox - -Run Functest Suite -================== - -Functest supports Doctor testing by triggering the test script above in a -Functest container. You can run the Doctor test with the following steps: - -.. code-block:: bash - - DOCKER_TAG=latest - docker pull docker.io/opnfv/functest-features:${DOCKER_TAG} - docker run --privileged=true -id \ - -e INSTALLER_TYPE=${INSTALLER_TYPE} \ - -e INSTALLER_IP=${INSTALLER_IP} \ - -e INSPECTOR_TYPE=sample \ - docker.io/opnfv/functest-features:${DOCKER_TAG} /bin/bash - docker exec <container_id> functest testcase run doctor-notification - -See `Functest Userguide`_ for more information. - -.. _Functest Userguide: :doc:`<functest:testing/user/userguide>` - - -For testing with stable version, change DOCKER_TAG to 'stable' or other release -tag identifier. - -Tips -==== + +Note! In DevStack you run Doctor in controller node. diff --git a/docs/testing/index.rst b/docs/testing/index.rst new file mode 100644 index 00000000..3fae9568 --- /dev/null +++ b/docs/testing/index.rst @@ -0,0 +1,15 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +.. _testing: + +======= +Testing +======= + +.. toctree:: + :maxdepth: 2 + + ./developer/index.rst + ./user/index.rst diff --git a/docs/testing/user/index.rst b/docs/testing/user/index.rst new file mode 100644 index 00000000..1be9c7eb --- /dev/null +++ b/docs/testing/user/index.rst @@ -0,0 +1,13 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +**** +User +**** + +.. toctree:: + :numbered: + :maxdepth: 2 + + testing.rst diff --git a/docs/testing/user/testing.rst b/docs/testing/user/testing.rst new file mode 100644 index 00000000..6172d26a --- /dev/null +++ b/docs/testing/user/testing.rst @@ -0,0 +1,30 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Run Functest Suite (obsolete) +============================= + +Functest supports Doctor testing by triggering the test script above in a +Functest container. You can run the Doctor test with the following steps: + +.. code-block:: bash + + DOCKER_TAG=latest + docker pull docker.io/opnfv/functest-features:${DOCKER_TAG} + docker run --privileged=true -id \ + -e INSTALLER_TYPE=${INSTALLER_TYPE} \ + -e INSTALLER_IP=${INSTALLER_IP} \ + -e INSPECTOR_TYPE=sample \ + docker.io/opnfv/functest-features:${DOCKER_TAG} /bin/bash + docker exec <container_id> functest testcase run doctor-notification + +See `Functest Userguide`_ for more information. + +.. _Functest Userguide: :doc:`<functest:testing/user/userguide>` + + +For testing with stable version, change DOCKER_TAG to 'stable' or other release +tag identifier. + +Tips +==== diff --git a/doctor_tests/admin_tool/fenix/Dockerfile b/doctor_tests/admin_tool/fenix/Dockerfile index 5804b207..202380eb 100644 --- a/doctor_tests/admin_tool/fenix/Dockerfile +++ b/doctor_tests/admin_tool/fenix/Dockerfile @@ -21,7 +21,7 @@ RUN apk --no-cache add ca-certificates && \ if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi && \ if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \ pip3 install --upgrade pip && \ - pip3 install alembic aodhclient decorator flask Flask-RESTful eventlet \ + pip3 install alembic aodhclient decorator flask Flask-RESTful eventlet jsonschema \ keystoneauth1 keystonemiddleware python-novaclient oslo.config pecan \ oslo.db oslo.log oslo.messaging oslo.serialization oslo.service oslo_policy \ oslotest oslo.utils pbr pymysql six sqlalchemy -cupper-constraints.txt && \ diff --git a/doctor_tests/app_manager/vnfm.py b/doctor_tests/app_manager/vnfm.py index 7a3c9add..68fdbb88 100644 --- a/doctor_tests/app_manager/vnfm.py +++ b/doctor_tests/app_manager/vnfm.py @@ -109,7 +109,7 @@ class VNFManager(Thread): if ep.service_id == maint_id and ep.interface == 'public'][0] self.log.info('maintenance endpoint: %s' % self.maint_endpoint) - + self.update_constraints_lock = False self.update_constraints() def delete_remote_instance_constraints(self, instance_id): @@ -153,6 +153,10 @@ class VNFManager(Thread): self.delete_remote_group_constraints(self.ha_group) def update_constraints(self): + while self.update_constraints_lock: + self.log.info('Waiting update_constraints_lock...') + time.sleep(1) + self.update_constraints_lock = True self.log.info('Update constraints') if self.project_id is None: self.project_id = self.keystone.projects.list( @@ -185,7 +189,7 @@ class VNFManager(Thread): "recovery_time": 4, "resource_mitigation": True} self.log.info('create doctor_ha_app_group constraints: %s' - % self.nonha_group) + % self.ha_group) self.update_remote_group_constraints(self.ha_group) instance_constraints = {} for ha_instance in self.ha_instances: @@ -195,7 +199,7 @@ class VNFManager(Thread): "group_id": self.ha_group["group_id"], "instance_name": ha_instance.name, "max_interruption_time": 120, - "migration_type": "MIGRATION", + "migration_type": "MIGRATE", "resource_mitigation": True, "lead_time": 40} self.log.info('create ha instance constraints: %s' @@ -208,7 +212,7 @@ class VNFManager(Thread): "group_id": self.nonha_group["group_id"], "instance_name": nonha_instance.name, "max_interruption_time": 120, - "migration_type": "MIGRATION", + "migration_type": "MIGRATE", "resource_mitigation": True, "lead_time": 40} self.log.info('create nonha instance constraints: %s' @@ -234,11 +238,12 @@ class VNFManager(Thread): for instance_id in deleted: self.delete_remote_instance_constraints(instance_id) updated = added + modified - for instance in [instance_constraints[i] in i in updated]: + for instance in [instance_constraints[i] for i in updated]: self.update_remote_instance_constraints(instance) if updated or deleted: # Some instance constraints have changed self.instance_constraints = instance_constraints.copy() + self.update_constraints_lock = False def active_instance_id(self): # Need rertry as it takes time after heat template done before @@ -358,14 +363,20 @@ class VNFManager(Thread): instance_ids = (self.get_session_instance_ids( payload['instance_ids'], payload['session_id'])) - reply['instance_ids'] = instance_ids - reply_state = 'ACK_MAINTENANCE' + my_instance_ids = self.get_instance_ids() + invalid_instances = ( + [instance_id for instance_id in instance_ids + if instance_id not in my_instance_ids]) + if invalid_instances: + self.log.error('Invalid instances: %s' % invalid_instances) + reply_state = 'NACK_MAINTENANCE' + else: + reply_state = 'ACK_MAINTENANCE' elif state == 'SCALE_IN': # scale down "self.scale" instances that is VCPUS equaling # at least a single compute node self.scale_instances(-self.scale) - reply['instance_ids'] = self.get_instance_ids() reply_state = 'ACK_SCALE_IN' elif state == 'MAINTENANCE_COMPLETE': @@ -411,7 +422,6 @@ class VNFManager(Thread): if reply_state: if self.conf.admin_tool.type == 'fenix': self.headers['X-Auth-Token'] = self.session.get_token() - reply['session_id'] = payload['session_id'] reply['state'] = reply_state url = payload['reply_url'] self.log.info('VNFM reply: %s' % reply) diff --git a/doctor_tests/installer/common/set_fenix.sh b/doctor_tests/installer/common/set_fenix.sh index 68bb4a6c..bd1eae47 100644 --- a/doctor_tests/installer/common/set_fenix.sh +++ b/doctor_tests/installer/common/set_fenix.sh @@ -22,14 +22,15 @@ apt-get install -y docker-ce docker-ce-cli containerd.io dpkg -r --force-depends golang-docker-credential-helpers } -docker ps | grep fenix >/dev/null && { -REMOTE=`docker exec -ti fenix git rev-parse origin/master` -LOCAL=`docker exec -ti fenix git rev-parse @` -if [ $LOCAL = $REMOTE ]; then - echo "Fenix start: Already running latest" +docker ps | grep fenix -q && { +REMOTE=`git ls-remote https://opendev.org/x/fenix HEAD | awk '{ print $1}'` +LOCAL=`docker exec -t fenix git rev-parse @` +if [[ "$LOCAL" =~ "$REMOTE" ]]; then + # Difference in above string ending marks, so cannot compare equal + echo "Fenix start: Already running latest $LOCAL equals $REMOTE" exit 0 else - echo "Fenix container needs to be recreated..." + echo "Fenix container needs to be recreated $LOCAL not $REMOTE" # Remove previous container for img in `docker image list | grep "^fenix" | awk '{print $1}'`; do for dock in `docker ps --all -f "ancestor=$img" | grep "$img" | awk '{print $1}'`; do @@ -75,7 +76,7 @@ echo "password = $OS_PASSWORD" >> fenix-api.conf echo "username = $OS_USERNAME" >> fenix-api.conf echo "cafile = /opt/stack/data/ca-bundle.pem" >> fenix-api.conf -openstack service list | grep maintenance | { +openstack service list | grep -q maintenance || { openstack service create --name fenix --enable maintenance openstack endpoint create --region $OS_REGION_NAME --enable fenix public http://localhost:12347/v1 } diff --git a/doctor_tests/installer/mcp.py b/doctor_tests/installer/mcp.py index f8f33c83..7659c9e2 100644 --- a/doctor_tests/installer/mcp.py +++ b/doctor_tests/installer/mcp.py @@ -129,8 +129,8 @@ class McpInstaller(BaseInstaller): def set_apply_patches(self): self.log.info('Set apply patches start......') fenix_files = None - set_scripts = [self.cm_set_script] + thrs = [] restart_cmd = 'sudo systemctl restart' \ ' ceilometer-agent-notification.service' @@ -152,10 +152,10 @@ class McpInstaller(BaseInstaller): 'admin_tool/fenix', fenix_file) client.scp(src_file, fenix_file) - self._run_apply_patches(client, - restart_cmd, - set_scripts, - python=self.python) + thrs.append(self._run_apply_patches(client, + restart_cmd, + set_scripts, + python=self.python)) time.sleep(5) self.log.info('Set apply patches start......') @@ -165,11 +165,15 @@ class McpInstaller(BaseInstaller): for node_ip in self.computes: client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file) - self._run_apply_patches(client, - restart_cmd, - [self.nc_set_compute_script], - python=self.python) + thrs.append(self._run_apply_patches( + client, + restart_cmd, + [self.nc_set_compute_script], + python=self.python)) time.sleep(5) + # If Fenix container ir build, it needs to be ready before continue + for thr in thrs: + thr.join() def restore_apply_patches(self): self.log.info('restore apply patches start......') |
