aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--ci/config_functest.yaml1
-rw-r--r--ci/config_patch.yaml1
-rwxr-xr-xci/exec_test.sh19
-rw-r--r--ci/testcases.yaml23
-rw-r--r--cli/commands/cli_os.py6
-rw-r--r--docker/Dockerfile2
-rw-r--r--docs/images/concepts_mapping_final.pngbin203419 -> 145191 bytes
-rw-r--r--docs/images/functest-reporting-status.pngbin0 -> 22859 bytes
-rw-r--r--docs/userguide/index.rst198
-rw-r--r--docs/userguide/introduction.rst52
-rw-r--r--docs/userguide/runfunctest.rst149
-rw-r--r--docs/userguide/troubleshooting.rst40
-rwxr-xr-xtestcases/Controllers/ODL/OpenDaylightTesting.py4
-rw-r--r--testcases/Controllers/ONOS/Sfc/Sfc_fun.py16
-rwxr-xr-xtestcases/OpenStack/healthcheck/healthcheck.sh5
-rw-r--r--testcases/OpenStack/tempest/custom_tests/blacklist.txt34
-rwxr-xr-xtestcases/OpenStack/tempest/gen_tempest_conf.py4
-rwxr-xr-xtestcases/OpenStack/tempest/run_tempest.py5
-rwxr-xr-xtestcases/features/copper.py171
-rwxr-xr-xtestcases/features/doctor.py44
-rwxr-xr-xtestcases/features/domino.py48
-rwxr-xr-xtestcases/features/sfc/set-up-tacker.sh6
-rwxr-xr-xtestcases/features/sfc/sfc.py18
-rw-r--r--testcases/security_scan/scripts/internet_check.py25
-rwxr-xr-xtestcases/security_scan/security_scan.py52
-rw-r--r--testcases/vnf/vRNC/parser.py5
-rw-r--r--utils/functest_utils.py16
27 files changed, 570 insertions, 374 deletions
diff --git a/ci/config_functest.yaml b/ci/config_functest.yaml
index 8e45b1d27..afa6ff1f6 100644
--- a/ci/config_functest.yaml
+++ b/ci/config_functest.yaml
@@ -50,6 +50,7 @@ general:
healthcheck:
disk_image: /home/opnfv/functest/data/cirros-0.3.4-x86_64-disk.img
+ disk_format: qcow2
vping:
ping_timeout: 200
diff --git a/ci/config_patch.yaml b/ci/config_patch.yaml
index 467a83ca0..4994d49d1 100644
--- a/ci/config_patch.yaml
+++ b/ci/config_patch.yaml
@@ -7,3 +7,4 @@ lxd:
healthcheck:
disk_image: /home/opnfv/functest/data/cirros-0.3.4-x86_64-lxc.tar.gz
+ disk_format: raw \ No newline at end of file
diff --git a/ci/exec_test.sh b/ci/exec_test.sh
index 9951c0973..98d54f205 100755
--- a/ci/exec_test.sh
+++ b/ci/exec_test.sh
@@ -36,8 +36,8 @@ if [[ "${CI_DEBUG,,}" == "true" ]];then
debug="--debug"
fi
-FUNCTEST_REPO_DIR=${repos_dir}/functest/
-FUNCTEST_CONF_DIR=/home/opnfv/functest/conf/
+FUNCTEST_REPO_DIR=${repos_dir}/functest
+FUNCTEST_CONF_DIR=/home/opnfv/functest/conf
function odl_tests(){
@@ -130,7 +130,7 @@ function run_test(){
sleep 10 # to let the instances terminate
;;
"doctor")
- python ${FUNCTEST_REPO_DIR}/testcases/features/doctor.py
+ python ${FUNCTEST_REPO_DIR}/testcases/features/doctor.py $report
;;
"ovno")
# suite under rewritting for colorado
@@ -140,23 +140,22 @@ function run_test(){
"security_scan")
echo "Sourcing Credentials ${FUNCTEST_CONF_DIR}/stackrc for undercloud .."
source ${FUNCTEST_CONF_DIR}/stackrc
- python ${FUNCTEST_REPO_DIR}/testcases/security_scan/security_scan.py --config ${FUNCTEST_REPO_DIR}/testcases/security_scan/config.ini
+ python ${repos_dir}/securityscanning/security_scan.py --config ${repos_dir}/securityscanning/config.ini
;;
"copper")
- python ${FUNCTEST_REPO_DIR}/testcases/features/copper.py
+ python ${FUNCTEST_REPO_DIR}/testcases/features/copper.py $report
;;
"moon")
- # TODO put the link to Moon script to be run (if test runnable) here
- ${repos_dir}/moon/tests/run_tests.sh
+ python ${repos_dir}/moon/tests/run_tests.py $report
;;
"multisite")
python ${FUNCTEST_REPO_DIR}/testcases/OpenStack/tempest/gen_tempest_conf.py
python ${FUNCTEST_REPO_DIR}/testcases/OpenStack/tempest/run_tempest.py \
- $clean_flag -s -m feature_multisite $report
+ $clean_flag -s -m feature_multisite $report \
-c ${FUNCTEST_REPO_DIR}/testcases/OpenStack/tempest/tempest_multisite.conf
;;
"domino")
- python ${FUNCTEST_REPO_DIR}/testcases/features/domino.py
+ python ${FUNCTEST_REPO_DIR}/testcases/features/domino.py $report
;;
"odl-sfc")
bash ${FUNCTEST_REPO_DIR}/testcases/features/sfc/server_presetup_CI.bash
@@ -170,7 +169,7 @@ function run_test(){
exit $ret_val
fi
source ${FUNCTEST_REPO_DIR}/testcases/features/sfc/tackerc
- python ${FUNCTEST_REPO_DIR}/testcases/features/sfc/sfc.py
+ python ${FUNCTEST_REPO_DIR}/testcases/features/sfc/sfc.py $report
;;
"parser")
python ${FUNCTEST_REPO_DIR}/testcases/vnf/vRNC/parser.py $report
diff --git a/ci/testcases.yaml b/ci/testcases.yaml
index 47d544b96..11056708d 100644
--- a/ci/testcases.yaml
+++ b/ci/testcases.yaml
@@ -47,7 +47,7 @@ tiers:
2) Connectivity between 2 instances over a private network.
dependencies:
installer: ''
- scenario: ''
+ scenario: '^((?!lxd).)*$'
-
name: tempest_smoke_serial
@@ -173,7 +173,7 @@ tiers:
scenario: '(odl)*(moon)'
-
name: multisite
- criteria: 'status == "PASS"'
+ criteria: 'success_rate == 100%'
blocking: false
description: >-
Test suite from kingbird
@@ -207,7 +207,15 @@ tiers:
dependencies:
installer: ''
scenario: 'onos-sfc'
-
+ -
+ name: parser
+ criteria: 'ret == 0'
+ blocking: false
+ description: >-
+ Test suite from Parser project.
+ dependencies:
+ installer: 'fuel'
+ scenario: ''
-
name: openstack
order: 4
@@ -255,12 +263,3 @@ tiers:
dependencies:
installer: ''
scenario: '(ocl)|(nosdn)|^(os-odl)((?!bgpvpn).)*$'
- -
- name: parser
- criteria: 'ret == 0'
- blocking: false
- description: >-
- Test suite from Parser project.
- dependencies:
- installer: 'fuel'
- scenario: ''
diff --git a/cli/commands/cli_os.py b/cli/commands/cli_os.py
index b007842b3..722c700cd 100644
--- a/cli/commands/cli_os.py
+++ b/cli/commands/cli_os.py
@@ -106,7 +106,8 @@ class CliOpenStack:
def snapshot_show(self):
if not os.path.isfile(OS_SNAPSHOT_FILE):
click.echo("There is no OpenStack snapshot created. To create "
- "one run the command 'functest env os-create-snapshot'")
+ "one run the command "
+ "'functest openstack snapshot-create'")
return
with open(OS_SNAPSHOT_FILE, 'r') as yaml_file:
click.echo("\n%s"
@@ -117,6 +118,7 @@ class CliOpenStack:
if not os.path.isfile(OS_SNAPSHOT_FILE):
click.echo("Not possible to clean OpenStack without a snapshot. "
"This could cause problems. "
- "Run first the command 'os-create-shapshot'.")
+ "Run first the command "
+ "'functest openstack snapshot-create'")
return
os_clean.main()
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 04ebf2c6d..ce3f4ce51 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -86,6 +86,7 @@ RUN git clone https://gerrit.opnfv.org/gerrit/sdnvpn ${repos_dir}/sdnvpn
RUN git clone https://github.com/openstack/kingbird.git ${repos_dir}/kingbird
RUN git clone https://gerrit.opnfv.org/gerrit/domino ${repos_dir}/domino
RUN git clone https://gerrit.opnfv.org/gerrit/parser ${repos_dir}/parser
+RUN git clone https://gerrit.opnfv.org/gerrit/securityscanning ${repos_dir}/securityscanning
RUN pip install -r ${repos_dir}/functest/docker/requirements.pip
RUN pip install -r ${repos_dir}/rally/requirements.txt
@@ -104,6 +105,7 @@ RUN curl -L https://get.rvm.io | bash -s stable
RUN /bin/bash -c ". /home/opnfv/repos/functest/testcases/features/sfc/tacker_client_install.sh"
RUN cd ${repos_dir}/bgpvpn && pip install .
RUN cd ${repos_dir}/kingbird && pip install -e .
+RUN cd ${repos_dir}/moon/moonclient/ && python setup.py install
RUN /bin/bash -c ". /etc/profile.d/rvm.sh \
&& cd /home/opnfv/repos/vims-test \
diff --git a/docs/images/concepts_mapping_final.png b/docs/images/concepts_mapping_final.png
index 7a51f1117..5ad0fc569 100644
--- a/docs/images/concepts_mapping_final.png
+++ b/docs/images/concepts_mapping_final.png
Binary files differ
diff --git a/docs/images/functest-reporting-status.png b/docs/images/functest-reporting-status.png
new file mode 100644
index 000000000..9e230df57
--- /dev/null
+++ b/docs/images/functest-reporting-status.png
Binary files differ
diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst
index 327a1756b..c0c3aa1b2 100644
--- a/docs/userguide/index.rst
+++ b/docs/userguide/index.rst
@@ -90,7 +90,7 @@ Tenant network::
| | to VM2 | |
| +------------------->| |
| | | |
- | | Stablish SSH | |
+ | | Establish SSH | |
| | connection to VM2 | |
| | through floating IP| |
| +------------------->| |
@@ -238,42 +238,41 @@ OpenDaylight and Neutron.
The list of tests can be described as follows:
- * Restconf.basic: Get the controller modules via Restconf
- * Neutron.Networks
+ * Basic Restconf test cases
+ * Connect to Restconf URL
+ * Check the HTTP code status
- * Check OpenStack Networks :: Checking OpenStack Neutron for known networks
- * Check OpenDaylight Networks :: Checking OpenDaylight Neutron API
- * Create Network :: Create new network in OpenStack
- * Check Network :: Check Network created in OpenDaylight
- * Neutron.Networks :: Checking Network created in OpenStack are pushed
+ * Neutron Reachability test cases
+ * Get the complete list of neutron resources (networks, subnets, ports)
- * Neutron.Subnets
+ * Neutron Network test cases
+ * Check OpenStack networks
+ * Check OpenDaylight networks
+ * Create a new network via OpenStack and check the HTTP status code returned by Neutron
+ * Check that the network has also been successfully created in OpenDaylight
- * Check OpenStack Subnets :: Checking OpenStack Neutron for known Subnets
- * Check OpenDaylight subnets :: Checking OpenDaylight Neutron API
- * Create New subnet :: Create new subnet in OpenStack
- * Check New subnet :: Check new subnet created in OpenDaylight
- * Neutron.Subnets :: Checking Subnets created in OpenStack are pushed
+ * Neutron Subnet test cases
+ * Check OpenStack subnets
+ * Check OpenDaylight subnets
+ * Create a new subnet via OpenStack and check the HTTP status code returned by Neutron
+ * Check that the subnet has also been successfully created in OpenDaylight
- * Neutron.Ports
+ * Neutron Port test cases
+ * Check OpenStack Neutron for known ports
+ * Check OpenDaylight ports
+ * Create a new port via OpenStack and check the HTTP status code returned by Neutron
+ * Check that the new port has also been successfully created in OpenDaylight
- * Check OpenStack ports :: Checking OpenStack Neutron for known ports
- * Check OpenDaylight ports :: Checking OpenDaylight Neutron API
- * Create New Port :: Create new port in OpenStack
- * Check New Port :: Check new subnet created in OpenDaylight
- * Neutron.Ports :: Checking Port created in OpenStack are pushed
+ * Delete operations
+ * Delete the port previously created via OpenStack
+ * Check that the port has been also succesfully deleted in OpenDaylight
+ * Delete previously subnet created via OpenStack
+ * Check that the subnet has also been successfully deleted in OpenDaylight
+ * Delete the network created via OpenStack
+ * Check that the network has also been succesfully deleted in OpenDaylight
- * Delete Ports
-
- * Delete previously created subnet in OpenStack
- * Check subnet deleted in OpenDaylight
- * Check subnet deleted in OpenStack
-
- * Delete network
-
- * Delete previously created network in OpenStack
- * Check network deleted in OpenDaylight
- * Check network deleted in OpenStack
+Note: the checks in OpenDaylight are based on the returned HTTP status
+code returned by OpenDaylight.
ONOS
@@ -320,70 +319,25 @@ The test cases are described as follows:
* Delete External Gateway: Delete the External Gateway and check that it is
'NULL' in ONOS
-Open Contrail
-^^^^^^^^^^^^^
-**TODO:**
-
-
Features
--------
-Doctor
-^^^^^^
-
-**TODO:**
-
-
-
-Promise
-^^^^^^^
-
-Promise provides a basic set of test cases as part of the deliverable.
-
-The available 33 test cases can be grouped into 7 test suites:
-
- #. Add a new OpenStack provider into resource pool: Registers
- OpenStack into a new resource pool and adds more capacity associated
- with this pool.
-
- #. Allocation without reservation: Creates a new server in OpenStack
- and adds a new allocation record in Promise shim-layer.
+Most of the features have been developped by feature projects.
+Security_scan has been initiated in Functest repository but should soon
+be declared in its own repository as well.
- #. Allocation using reservation for immediate use: Creates a resource
- reservation record with no start/end time and immediately creates a new
- server in OpenStack and add a new allocation record in Promise
- shim-layer.
+Please refer to the dedicated feature user guides for details:
- #. Reservation for future use: Creates a resource reservation record
- for a future start time, queries, modifies and cancels the newly created
- reservation.
-
- #. Capacity planning: Decreases and increases the available capacity
- from a provider in the future and queries the available collections and
- utilizations.
-
- #. Reservation with conflict: Tries to create reservations for
- immediate and future use with conflict.
-
- #. Cleanup test allocations: Destroys all allocations in OpenStack.
-
-
-bgpvpn
-^^^^^^
-Many telecom network functions are relying on layer-3 infrastructure services,
-within a VNF between components, or towards existing external networks.
-In many cases, these external networks are implemented in MPLS/BGP technology in
-existing service provider wide-area-networks (WAN). This proven technology
-provides a good mechanism for inter-operation of a NFV Infrastructure (NFVI)
-and WAN.
-
-The SDNVPN project defined a 'bgpvpn' test suite.
-This 'bgpvpn' test suite deals with 2 Tempest cases dedicated to the test of
-the OpenStack 'bgpvpn' API:
-
- * test_create_bgpvpn
- * test_create_bgpvpn_as_non_admin_fail
+ * bgpvpn: ** TODO link **
+ * copper: ** TODO link **
+ * doctor: ** TODO link **
+ * domino: ** TODO link **
+ * moon: ** TODO link **
+ * multisites: ** TODO link **
+ * onos-sfc: ** TODO link **
+ * odl-sfc: ** TODO link **
+ * promise: ** TODO link **
security_scan
^^^^^^^^^^^^^
@@ -418,6 +372,8 @@ The current work flow is as follows:
At present, only the Apex installer is supported, with support for other installers due
within D-release.
+
+
VNF
---
@@ -446,16 +402,71 @@ The Clearwater architecture is described as follows:
:alt: vIMS architecture
+parser
+^^^^^^
+
+See parser user guide for details: ** TODO link **
+
+
.. include:: ./runfunctest.rst
Test results
============
-Note that the results are documented per scenario basis. Although most of the test
-cases might show the same output, some of them are not supported by
-certain scenarios. Please select the appropriate scenario and compare the results
-to that referenced in the documentation.
+Manual testing
+--------------
+
+In manual mode test results are displayed in the console and result files
+are put in /home/opnfv/functest/results.
+
+Automated testing
+--------------
+
+In automated mode, test results are displayed in jenkins logs, a summary is provided
+at the end of the job and can be described as follow::
+
+ +==================================================================================================================================================+
+ | FUNCTEST REPORT |
+ +==================================================================================================================================================+
+ | |
+ | Deployment description: |
+ | INSTALLER: fuel |
+ | SCENARIO: os-odl_l2-nofeature-ha |
+ | BUILD TAG: jenkins-functest-fuel-baremetal-daily-master-324 |
+ | CI LOOP: daily |
+ | |
+ +=========================+===============+============+===============+===========================================================================+
+ | TEST CASE | TIER | DURATION | RESULT | URL |
+ +=========================+===============+============+===============+===========================================================================+
+ | healthcheck | healthcheck | 03:07 | PASS | |
+ +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+
+ | vping_ssh | smoke | 00:56 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac13d79377c54b278bd4c1 |
+ +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+
+ | vping_userdata | smoke | 00:41 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac14019377c54b278bd4c2 |
+ +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+
+ | tempest_smoke_serial | smoke | 16:05 | FAIL | http://testresults.opnfv.org/test/api/v1/results/57ac17ca9377c54b278bd4c3 |
+ +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+
+ | rally_sanity | smoke | 12:19 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac1aad9377c54b278bd4cd |
+ +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+
+ | odl | sdn_suites | 00:24 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac1ad09377c54b278bd4ce |
+ +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+
+ | promise | features | 00:41 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac1ae59377c54b278bd4cf |
+ +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+
+
+Results are automatically pushed to the test results database, some additional
+result files are pushed to OPNFV artifact web sites.
+
+Based on the results stored in the result database, a `Functest reporting`_
+portal is also automatically updated. This portal provides information on:
+
+ * The overall status per scenario and per installer
+ * Tempest: Tempest test case including reported errors per scenario and installer
+ * vIMS: vIMS details per scenario and installer
+
+.. figure:: ../images/functest-reporting-status.png
+ :align: center
+ :alt: Functest reporting portal Fuel status page
Test Dashboard
@@ -492,3 +503,4 @@ IRC support chan: #opnfv-testperf
.. _`Rally installation procedure`: https://rally.readthedocs.org/en/latest/tutorial/step_0_installation.html
.. _`config_test.py` : https://git.opnfv.org/cgit/functest/tree/testcases/config_functest.py
.. _`config_functest.yaml` : https://git.opnfv.org/cgit/functest/tree/testcases/config_functest.yaml
+.. _`Functest reporting`: http://testresults.opnfv.org/reporting/functest/release/master/index-status-fuel.html
diff --git a/docs/userguide/introduction.rst b/docs/userguide/introduction.rst
index 8624bee83..ce84a9c43 100644
--- a/docs/userguide/introduction.rst
+++ b/docs/userguide/introduction.rst
@@ -58,7 +58,11 @@ Features and VNF (Virtual Network Functions).
| | | | See the Rally documents `[3]`_. |
+-------------+---------------+----------------+----------------------------------+
| Controllers | sdn_suites | odl | Opendaylight Test suite |
-| | | | TODO: Find a document reference! |
+| | | | Limited test suite to check the |
+| | | | basic neutron (Layer 2) |
+| | | | operations mainly based on |
+| | | | upstream testcases. See below |
+| | | | for details |
| | +----------------+----------------------------------+
| | | onos | Test suite of ONOS L2 and L3 |
| | | | functions. |
@@ -77,11 +81,13 @@ Features and VNF (Virtual Network Functions).
| | | | details. |
| | +----------------+----------------------------------+
| | | Doctor | Doctor platform, as of Colorado |
-| | | | release, provides the two |
+| | | | release, provides the three |
| | | | features: |
| | | | * Immediate Notification |
| | | | * Consistent resource state |
-| | | | awareness (compute). See the |
+| | | | awareness for compute host down |
+| | | | * Valid compute host status |
+| | | | given to VM owner |
| | | | See `Doctor User Guide`_ for |
| | | | details |
| | +----------------+----------------------------------+
@@ -96,8 +102,30 @@ Features and VNF (Virtual Network Functions).
| | | | security scan. (Currently |
| | | | available only for the Apex |
| | | | installer environment) |
-| | | | TODO: Add document link from |
-| | | | Luke Hinds; when received. |
+| | +----------------+----------------------------------+
+| | | onos-sfc | SFC testing for onos scenarios |
+| | | | TODO See for details |
+| | +----------------+----------------------------------+
+| | | odl-sfc | SFC testing for odl scenarios |
+| | | | TODO See for details |
+| | +----------------+----------------------------------+
+| | | domino | Domino provides TOSCA template |
+| | | | distribution service for network |
+| | | | service and VNF descriptors |
+| | | | among MANO components e.g., |
+| | | | NFVO, VNFM, VIM, SDN-C, etc., |
+| | | | as well as OSS/BSS functions. |
+| | | | See `Domino User Guide`_ for |
+| | | | details |
+| | +----------------+----------------------------------+
+| | | copper | Deployment policy |
+| | | | TODO See for details |
+| | +----------------+----------------------------------+
+| | | multisites | Multisites |
+| | | | TODO See for details |
+| | +----------------+----------------------------------+
+| | | moon | Security management system |
+| | | | TODO See for details |
+-------------+---------------+----------------+----------------------------------+
| VNF | vnf | vims | Example of a real VNF deployment |
| | | | to show the NFV capabilities of |
@@ -105,7 +133,10 @@ Features and VNF (Virtual Network Functions).
| | | | Subsytem is a typical Telco test |
| | | | case, referenced by ETSI. |
| | | | It provides a fully functional |
-| | | | VoIP System, |
+| | | | VoIP System |
++ +---------------+----------------+----------------------------------+
+| | | parser | Service template translation |
+| | | | TODO See for details |
+-------------+---------------+----------------+----------------------------------+
@@ -129,8 +160,8 @@ are integrated from upstream communities or other OPNFV projects. For example,
OpenStack integration test suite and Functest is in charge of the selection,
integration and automation of those tests that fit suitably to OPNFV.
-The Tempest test suite has been customized but no new test cases have been
-created in OPNFV Functest.
+The Tempest test suite is the default OpenStack smoke test suite but no new test
+cases have been created in OPNFV Functest.
The results produced by the tests run from CI are pushed and collected into a
NoSQL database. The goal is to populate the database with results from different
@@ -176,6 +207,5 @@ section `Executing the functest suites`_ of this document.
.. _`Promise User Guide`: http://artifacts.opnfv.org/promise/brahmaputra/docs/userguide/index.html
.. _`ONOSFW User Guide`: http://artifacts.opnfv.org/onosfw/brahmaputra/docs/userguide/index.html
.. _`SDNVPN User Guide`: http://artifacts.opnfv.org/sdnvpn/brahmaputra/docs/userguide/featureusage.html
-.. _`Functest Dashboard`: http://testresults.opnfv.org/dashboard/
-
-
+.. _`Domino User Guide`: http://artifacts.opnfv.org/domino/docs/userguide/index.html
+.. _`Functest Dashboard`: http://testresults.opnfv.org/kibana_dashboards/
diff --git a/docs/userguide/runfunctest.rst b/docs/userguide/runfunctest.rst
index 4c2e3f54b..f3da540a4 100644
--- a/docs/userguide/runfunctest.rst
+++ b/docs/userguide/runfunctest.rst
@@ -38,8 +38,6 @@ for the execution of Test Tiers or Test Cases::
show Shows information about a tier.
root@22e436918db0:~/repos/functest/ci# functest testcase --help
- and
-
Usage: functest testcase [OPTIONS] COMMAND [ARGS]...
Options:
@@ -217,7 +215,7 @@ command is used with 'functest tier'::
Please note that for some scenarios some test cases might not be launched.
For example, the last example displayed only the 'odl' testcase for the given
-environment. In this system the deployment does not support the 'onos' SDN
+environment. In this particular system the deployment does not support the 'onos' SDN
Controller Test Case; for example.
**Important** If you use the command 'functest tier run <tier_name>', then the
@@ -264,12 +262,12 @@ Example::
:
etc.
-Functest includes cleaning mechanism in order to remove all the OpenStack
-resources except what was present before running any test. The script
+Functest includes a cleaning mechanism in order to remove all the OpenStack
+resources except those present before running any test. The script
*$repos_dir/functest/utils/generate_defaults.py* is called once when setting up
the Functest environment (i.e. CLI command 'functest env prepare') to snapshot
all the OpenStack resources (images, networks, volumes, security groups, tenants,
-users) so that an eventual cleanup does not remove any of this defaults.
+users) so that an eventual cleanup does not remove any of these defaults.
The script **clean_openstack.py** which is located in
*$repos_dir/functest/utils/* is normally called after a test execution. It is
@@ -297,21 +295,23 @@ do a direct call to the desired test script. For example:
Automated testing
-----------------
-**TODO** Jose, the next section has not yet been revised!
-
As mentioned previously, the Functest Docker container preparation as well as
invocation of Test Cases can be called within the container from the Jenkins CI
-system. There are 2 jobs that automate the whole process. The first job runs all
-the tests and the second job allows testing test suite by test suite specifying
+system. There are 3 jobs that automate the whole process. The first job runs all
+the tests referenced in the daily loop (i.e. that must been run daily), the second
+job runs the tests referenced in the weekly loop (usually long duration tests run
+once a week maximum) and the third job allows testing test suite by test suite specifying
the test suite name. The user may also use either of these Jenkins jobs to execute
the desired test suites.
-One of the most challenging task in the Brahmaputra release consists
+One of the most challenging task in the Colorado release consists
in dealing with lots of scenarios and installers. Thus, when the tests are
automatically started from CI, a basic algorithm has been created in order to
detect whether a given test is runnable or not on the given scenario.
Some Functest test suites cannot be systematically run (e.g. ODL suite can not
-be run on an ONOS scenario).
+be run on an ONOS scenario). Moreover since Colorado, we also introduce the
+notion of daily/weekly in order to save CI time and avoid running systematically
+long duration tests.
CI provides some useful information passed to the container as environment
variables:
@@ -320,65 +320,86 @@ variables:
* Installer IP of the engine or VM running the actual deployment, stored in INSTALLER_IP
* The scenario [controller]-[feature]-[mode], stored in DEPLOY_SCENARIO with
- * controller = (odl|onos|ocl|nosdn)
- * feature = (ovs(dpdk)|kvm|sfc|bgpvpn)
- * mode = (ha|noha)
+ * controller = (odl|onos|ocl|nosdn)
+ * feature = (ovs(dpdk)|kvm|sfc|bgpvpn|moon|multisites)
+ * mode = (ha|noha)
The constraints per test case are defined in the Functest configuration file
-*/home/opnfv/functest/config/config_functest.yaml*::
-
- test-dependencies:
- functest:
- vims:
- scenario: '(ocl)|(odl)|(nosdn)'
- vping:
- vping_userdata:
- scenario: '(ocl)|(odl)|(nosdn)'
- tempest:
- rally:
- odl:
- scenario: 'odl'
- onos:
- scenario: 'onos'
+*/home/opnfv/repos/functest/ci/testcases.yaml*::
+
+ tiers:
+ -
+ name: healthcheck
+ order: 0
+ ci_loop: '(daily)|(weekly)'
+ description : >-
+ First tier to be executed to verify the basic
+ operations in the VIM.
+ testcases:
+ -
+ name: healthcheck
+ criteria: 'status == "PASS"'
+ blocking: true
+ description: >-
+ This test case verifies the basic OpenStack services like
+ Keystone, Glance, Cinder, Neutron and Nova.
+
+ dependencies:
+ installer: ''
+ scenario: ''
+
+ -
+ name: smoke
+ order: 1
+ ci_loop: '(daily)|(weekly)'
+ description : >-
+ Set of basic Functional tests to validate the OpenStack deployment.
+ testcases:
+ -
+ name: vping_ssh
+ criteria: 'status == "PASS"'
+ blocking: true
+ description: >-
+ This test case verifies: 1) SSH to an instance using floating
+ IPs over the public network. 2) Connectivity between 2 instances
+ over a private network.
+ dependencies:
+ installer: ''
+ scenario: '^((?!bgpvpn|odl_l3).)*$'
....
-At the end of the Functest environment creation, a file
-*/home/opnfv/functest/conf/testcase-list.txt* is created with the list of
-all the runnable tests.
-Functest considers the static constraints as regular expressions and compare them
-with the given scenario name.
-For instance, ODL suite can be run only on an scenario including 'odl' in its name.
-
-The order of execution is also described in the Functest configuration file::
-
- test_exec_priority:
-
- 1: vping_ssh
- 2: vping_userdata
- 3: tempest
- 4: odl
- 5: onos
- 6: ovno
- 7: doctor
- 8: promise
- 9: odl-vpnservice
- 10: bgpvpn
- #11: openstack-neutron-bgpvpn-api-extension-tests
- 12: vims
- 13: rally
-
-The tests are executed in the following order:
-
- 1) vPing test cases
- 2) Tempest suite
- 3) SDN controller suites
- 4) Feature project tests cases (Promise, Doctor, BGPVPN...)
+We may distinguish 2 levels in the test case description:
+ * Tier level
+ * Test case level
+
+At the tier level, we define the following parameters:
+
+ * ci_loop: indicate if in automated mode, the test case must be run in daily and/or weekly jobs
+ * description: a high level view of the test case
+
+For a given test case we defined:
+ * the name of the test case
+ * the criteria (experimental): a criteria used to declare the test case as PASS or FAIL
+ * blocking: if set to true, if the test is failed, the execution of the following tests is canceled
+ * the description of the test case
+ * the dependencies: a combination of 2 regex on the scenario and the installer name
+
+The order of execution is the one defined in the file if all test cases are selected.
+
+In CI daily job the tests are executed in the following order:
+
+ 1) healthcheck (blocking)
+ 2) smoke: both vPings are blocking
+ 3) SDN controller suites (blocking)
+ 4) Feature project tests cases
+
+In CI weekly job we add 2 tiers:
+
5) vIMS suite
6) Rally suite
As explained before, at the end of an automated execution, the OpenStack resources
might be eventually removed.
+Please note that a system snapshot is taken before any test case execution.
-**END of TODO**
-
-
+This testcase.yaml file is used for CI, for the CLI and for the automatic reporting.
diff --git a/docs/userguide/troubleshooting.rst b/docs/userguide/troubleshooting.rst
index cd79bafcf..c23b00602 100644
--- a/docs/userguide/troubleshooting.rst
+++ b/docs/userguide/troubleshooting.rst
@@ -312,15 +312,19 @@ To check any possible problems with Rally, the logs are stored under
Controllers
-----------
-ODL
-^^^
-2 versions are supported in Colorado, depending on the scenario:
- * Lithium
- * Berylium
+Opendaylight
+^^^^^^^^^^^^
+
+If the Basic Restconf test suite fails, check that the ODL controller is
+reachable and its Restconf module has been installed.
+
+If the Neutron Reachability test fails, verify that the modules
+implementing Neutron requirements have been properly installed.
+
+If any of the other test cases fails, check that Neutron and ODL have
+been correctly configured to work together. Check Neutron configuration
+files, accounts, IP addresses etc.).
-The upstream test suites have not been adapted, so you may get 18 or 15 tests
-passed on 18 depending on your configuration. The 3 testcases are partly failed
-due to wrong return code.
ONOS
^^^^
@@ -331,20 +335,14 @@ Please refer to the ONOS documentation. `ONOSFW User Guide`_ .
Features
--------
-
-Doctor
-^^^^^^
-Please refer to the Doctor documentation. `Doctor User Guide`_
+Please refer to the dedicated feature user guides for details.
-Promise
-^^^^^^^
-Please refer to the Promise documentation. `Promise User Guide`_
+security_scan
+^^^^^^^^^^^^^
+** TODO **
-bgpvpn
-^^^^^^
-Please refer to the SNVPN documentation. `SDNVPN User Guide`_
NFV
@@ -383,4 +381,10 @@ described in the following table:
+-----------------------------------+------------------------------------+
+parser
+^^^^^^
+
+See parser user guide for details: ** TODO link **
+
+
.. _`OPNFV Functest Developer Guide`: http://artifacts.opnfv.org/functest/docs/devguide/#
diff --git a/testcases/Controllers/ODL/OpenDaylightTesting.py b/testcases/Controllers/ODL/OpenDaylightTesting.py
index db7b94982..79e385a2d 100755
--- a/testcases/Controllers/ODL/OpenDaylightTesting.py
+++ b/testcases/Controllers/ODL/OpenDaylightTesting.py
@@ -6,9 +6,11 @@ import os
import re
import shutil
import sys
-import functest.utils.functest_logger as ft_logger
+
from robot import run
+import functest.utils.functest_logger as ft_logger
+
class ODLTestCases:
diff --git a/testcases/Controllers/ONOS/Sfc/Sfc_fun.py b/testcases/Controllers/ONOS/Sfc/Sfc_fun.py
index b94eedcaf..0d9eaf80d 100644
--- a/testcases/Controllers/ONOS/Sfc/Sfc_fun.py
+++ b/testcases/Controllers/ONOS/Sfc/Sfc_fun.py
@@ -227,6 +227,22 @@ class Sfc_fun:
else:
return(response.status_code)
+ url = 'http://' + self.nova_hostname + \
+ ':8774/v2.1/' + self.tenant_id + '/flavors?name=m1.tiny'
+ headers = {"Accept": "application/json", "Content-Type":
+ "application/json", "X-Auth-Token": self.token_id}
+ response = requests.get(url, headers=headers)
+
+ if (response.status_code == 200):
+ self.logger.debug(response.status_code)
+ self.logger.debug(response.content)
+ self.logger.info("\tFlavor is available")
+ json1_data = json.loads(response.content)
+ self.logger.debug(json1_data)
+ self.flavorRef = json1_data['flavors'][0]['id']
+ else:
+ return(response.status_code)
+
for y in range(0, 3):
Dicdata = {}
org_nw_port = []
diff --git a/testcases/OpenStack/healthcheck/healthcheck.sh b/testcases/OpenStack/healthcheck/healthcheck.sh
index 2449ac93b..996aadcf7 100755
--- a/testcases/OpenStack/healthcheck/healthcheck.sh
+++ b/testcases/OpenStack/healthcheck/healthcheck.sh
@@ -120,6 +120,7 @@ info "...Keystone OK!"
info "Testing Glance API..."
#################################
disk_img=$(cat ${YAML_FILE} | shyaml get-value healthcheck.disk_image 2> /dev/null || true)
+disk_format=$(cat ${YAML_FILE} | shyaml get-value healthcheck.disk_format 2> /dev/null || true)
kernel_img=$(cat ${YAML_FILE} | shyaml get-value healthcheck.kernel_image 2> /dev/null || true)
ramdisk_img=$(cat ${YAML_FILE} | shyaml get-value healthcheck.ramdisk_image 2> /dev/null || true)
extra_properties=$(cat ${YAML_FILE} | shyaml get-value healthcheck.extra_properties 2> /dev/null || true)
@@ -151,10 +152,10 @@ fi
debug "image extra_properties=${extra_properties}"
-eval glance image-create --name ${image_1} --disk-format qcow2 --container-format bare \
+eval glance image-create --name ${image_1} --disk-format ${disk_format} --container-format bare \
${extra_opts} < ${disk_img}
debug "image '${image_1}' created."
-eval glance image-create --name ${image_2} --disk-format qcow2 --container-format bare \
+eval glance image-create --name ${image_2} --disk-format ${disk_format} --container-format bare \
${extra_opts} < ${disk_img}
debug "image '${image_2}' created."
info "... Glance OK!"
diff --git a/testcases/OpenStack/tempest/custom_tests/blacklist.txt b/testcases/OpenStack/tempest/custom_tests/blacklist.txt
index 5dd8fe2e1..6dd7fad5c 100644
--- a/testcases/OpenStack/tempest/custom_tests/blacklist.txt
+++ b/testcases/OpenStack/tempest/custom_tests/blacklist.txt
@@ -16,18 +16,48 @@
- tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern
- tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern
+-
scenarios:
- os-odl_l2-nofeature-ha
- os-odl_l2-nofeature-noha
- os-nosdn-nofeature-ha
- os-nosdn-nofeature-noha
+ - os-nosdn-lxd-ha
+ - os-nosdn-lxd-noha
+ installers:
+ - joid
+ tests:
+ - tempest.api.object_storage
+
+-
+ scenarios:
+ - os-onos-nofeature-ha
+ - os-onos-nofeature-noha
+ - os-onos-sfc-ha
+ - os-onos-sfc-noha
+ installers:
+ - fuel
+ - apex
+ - compass
+ tests:
+ - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard
+ - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops
+ - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops
+ - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern
+ - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern
+
+-
+ scenarios:
- os-onos-nofeature-ha
- os-onos-nofeature-noha
- os-onos-sfc-ha
- os-onos-sfc-noha
- - os-nosdn-lxd-ha
- - os-nosdn-lxd-noha
installers:
- joid
tests:
- tempest.api.object_storage
+ - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard
+ - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops
+ - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops
+ - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern
+ - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern
diff --git a/testcases/OpenStack/tempest/gen_tempest_conf.py b/testcases/OpenStack/tempest/gen_tempest_conf.py
index 79236b958..4aa814a51 100755
--- a/testcases/OpenStack/tempest/gen_tempest_conf.py
+++ b/testcases/OpenStack/tempest/gen_tempest_conf.py
@@ -17,6 +17,7 @@ import shutil
import functest.utils.functest_utils as ft_utils
import functest.utils.functest_logger as ft_logger
from run_tempest import configure_tempest
+from run_tempest import TEMPEST_RESULTS_DIR
logger = ft_logger.Logger("multisite").getLogger()
@@ -112,6 +113,9 @@ def configure_tempest_multisite(deployment_dir):
def main():
+ if not os.path.exists(TEMPEST_RESULTS_DIR):
+ os.makedirs(TEMPEST_RESULTS_DIR)
+
deployment_dir = ft_utils.get_deployment_dir(logger)
configure_tempest_multisite(deployment_dir)
diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py
index 3a8d8d72a..c143fffae 100755
--- a/testcases/OpenStack/tempest/run_tempest.py
+++ b/testcases/OpenStack/tempest/run_tempest.py
@@ -365,6 +365,8 @@ def run_tempest(OPTION):
if 'smoke' in args.mode:
case_name = 'tempest_smoke_serial'
+ elif 'feature' in args.mode:
+ case_name = args.mode.replace("feature_", "")
else:
case_name = 'tempest_full_parallel'
@@ -422,7 +424,8 @@ def main():
deployment_dir = ft_utils.get_deployment_dir(logger)
- if "" != args.conf:
+ if "" == args.conf:
+ MODE = ""
configure_tempest(deployment_dir)
else:
MODE = " --tempest-config " + args.conf
diff --git a/testcases/features/copper.py b/testcases/features/copper.py
index 9efcbd7f4..c312643eb 100755
--- a/testcases/features/copper.py
+++ b/testcases/features/copper.py
@@ -1,86 +1,85 @@
-#!/usr/bin/python
-#
-# Copyright 2016 AT&T Intellectual Property, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import sys
-import time
-import functest.utils.functest_logger as ft_logger
-import functest.utils.functest_utils as functest_utils
-import yaml
-
-
-with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
- functest_yaml = yaml.safe_load(f)
-
-dirs = functest_yaml.get('general').get('directories')
-FUNCTEST_REPO = dirs.get('dir_repo_functest')
-COPPER_REPO = dirs.get('dir_repo_copper')
-TEST_DB_URL = functest_yaml.get('results').get('test_db_url')
-
-logger = ft_logger.Logger("copper").getLogger()
-
-
-def main():
- cmd = "%s/tests/run.sh %s/tests" % (COPPER_REPO, COPPER_REPO)
-
- start_time = time.time()
-
- ret_val = functest_utils.execute_command(cmd, logger, exit_on_error=False)
-
- stop_time = time.time()
- duration = round(stop_time - start_time, 1)
- if ret_val == 0:
- logger.info("COPPER PASSED")
- test_status = 'PASS'
- else:
- logger.info("COPPER FAILED")
- test_status = 'FAIL'
-
- details = {
- 'timestart': start_time,
- 'duration': duration,
- 'status': test_status,
- }
- pod_name = functest_utils.get_pod_name(logger)
- scenario = functest_utils.get_scenario(logger)
- version = functest_utils.get_version(logger)
- build_tag = functest_utils.get_build_tag(logger)
-
- logger.info("Pushing COPPER results: TEST_DB_URL=%(db)s pod_name=%(pod)s "
- "version=%(v)s scenario=%(s)s criteria=%(c)s details=%(d)s" % {
- 'db': TEST_DB_URL,
- 'pod': pod_name,
- 'v': version,
- 's': scenario,
- 'c': details['status'],
- 'b': build_tag,
- 'd': details,
- })
- functest_utils.push_results_to_db("COPPER",
- "COPPER-notification",
- logger,
- start_time,
- stop_time,
- details['status'],
- details)
- if ret_val != 0:
- sys.exit(-1)
-
- sys.exit(0)
-
-if __name__ == '__main__':
- main()
+#!/usr/bin/python
+#
+# Copyright 2016 AT&T Intellectual Property, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import argparse
+import os
+import sys
+import time
+import functest.utils.functest_logger as ft_logger
+import functest.utils.functest_utils as functest_utils
+import yaml
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-r", "--report",
+ help="Create json result file",
+ action="store_true")
+args = parser.parse_args()
+
+with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
+ functest_yaml = yaml.safe_load(f)
+
+dirs = functest_yaml.get('general').get('directories')
+FUNCTEST_REPO = dirs.get('dir_repo_functest')
+COPPER_REPO = dirs.get('dir_repo_copper')
+
+logger = ft_logger.Logger("copper").getLogger()
+
+
+def main():
+ cmd = "%s/tests/run.sh %s/tests" % (COPPER_REPO, COPPER_REPO)
+
+ start_time = time.time()
+
+ ret_val = functest_utils.execute_command(cmd, logger, exit_on_error=False)
+
+ stop_time = time.time()
+ duration = round(stop_time - start_time, 1)
+ if ret_val == 0:
+ logger.info("COPPER PASSED")
+ test_status = 'PASS'
+ else:
+ logger.info("COPPER FAILED")
+ test_status = 'FAIL'
+
+ details = {
+ 'timestart': start_time,
+ 'duration': duration,
+ 'status': test_status,
+ }
+ functest_utils.logger_test_results(logger, "Copper",
+ "copper-notification",
+ details['status'], details)
+ try:
+ if args.report:
+ functest_utils.push_results_to_db("copper",
+ "copper-notification",
+ logger,
+ start_time,
+ stop_time,
+ details['status'],
+ details)
+ logger.info("COPPER results pushed to DB")
+ except:
+ logger.error("Error pushing results into Database '%s'"
+ % sys.exc_info()[0])
+
+ if ret_val != 0:
+ sys.exit(-1)
+
+ sys.exit(0)
+
+if __name__ == '__main__':
+ main()
diff --git a/testcases/features/doctor.py b/testcases/features/doctor.py
index 184ab032b..154cfc695 100755
--- a/testcases/features/doctor.py
+++ b/testcases/features/doctor.py
@@ -13,7 +13,7 @@
# 0.2: measure test duration and publish results under json format
#
#
-
+import argparse
import os
import time
import yaml
@@ -21,13 +21,18 @@ import yaml
import functest.utils.functest_logger as ft_logger
import functest.utils.functest_utils as functest_utils
+parser = argparse.ArgumentParser()
+parser.add_argument("-r", "--report",
+ help="Create json result file",
+ action="store_true")
+args = parser.parse_args()
+
with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
functest_yaml = yaml.safe_load(f)
dirs = functest_yaml.get('general').get('directories')
FUNCTEST_REPO = dirs.get('dir_repo_functest')
DOCTOR_REPO = dirs.get('dir_repo_doctor')
-TEST_DB_URL = functest_yaml.get('results').get('test_db_url')
logger = ft_logger.Logger("doctor").getLogger()
@@ -55,32 +60,21 @@ def main():
'duration': duration,
'status': test_status,
}
- pod_name = functest_utils.get_pod_name(logger)
- scenario = functest_utils.get_scenario(logger)
- version = functest_utils.get_version(logger)
- build_tag = functest_utils.get_build_tag(logger)
-
status = "FAIL"
if details['status'] == "OK":
status = "PASS"
-
- logger.info("Pushing Doctor results: TEST_DB_URL=%(db)s pod_name=%(pod)s "
- "version=%(v)s scenario=%(s)s criteria=%(c)s details=%(d)s" % {
- 'db': TEST_DB_URL,
- 'pod': pod_name,
- 'v': version,
- 's': scenario,
- 'c': status,
- 'b': build_tag,
- 'd': details,
- })
- functest_utils.push_results_to_db("doctor",
- "doctor-notification",
- logger,
- start_time,
- stop_time,
- status,
- details)
+ functest_utils.logger_test_results(logger, "Doctor",
+ "doctor-notification",
+ status, details)
+ if args.report:
+ functest_utils.push_results_to_db("doctor",
+ "doctor-notification",
+ logger,
+ start_time,
+ stop_time,
+ status,
+ details)
+ logger.info("Doctor results pushed to DB")
exit(exit_code)
diff --git a/testcases/features/domino.py b/testcases/features/domino.py
index e4229b4c7..03ad4a2af 100755
--- a/testcases/features/domino.py
+++ b/testcases/features/domino.py
@@ -11,9 +11,10 @@
# Later, the VM2 boots then execute cloud-init to ping VM1.
# After successful ping, both the VMs are deleted.
# 0.2: measure test duration and publish results under json format
-#
+# 0.3: add report flag to push results when needed
#
+import argparse
import os
import time
import yaml
@@ -21,13 +22,19 @@ import yaml
import functest.utils.functest_logger as ft_logger
import functest.utils.functest_utils as functest_utils
+parser = argparse.ArgumentParser()
+
+parser.add_argument("-r", "--report",
+ help="Create json result file",
+ action="store_true")
+args = parser.parse_args()
+
with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
functest_yaml = yaml.safe_load(f)
dirs = functest_yaml.get('general').get('directories')
FUNCTEST_REPO = dirs.get('dir_repo_functest')
DOMINO_REPO = dirs.get('dir_repo_domino')
-TEST_DB_URL = functest_yaml.get('results').get('test_db_url')
logger = ft_logger.Logger("domino").getLogger()
@@ -55,10 +62,6 @@ def main():
'duration': duration,
'status': test_status,
}
- pod_name = functest_utils.get_pod_name(logger)
- scenario = functest_utils.get_scenario(logger)
- version = functest_utils.get_version(logger)
- build_tag = functest_utils.get_build_tag(logger)
status = "FAIL"
if details['status'] == "OK":
@@ -66,25 +69,20 @@ def main():
elif details['status'] == "SKIPPED":
status = "SKIP"
- logger.info("Pushing Domino results: TEST_DB_URL=%(db)s pod_name=%(pod)s "
- "version=%(v)s scenario=%(s)s criteria=%(c)s details=%(d)s" % {
- 'db': TEST_DB_URL,
- 'pod': pod_name,
- 'v': version,
- 's': scenario,
- 'c': status,
- 'b': build_tag,
- 'd': details,
- })
-
- if status is not "SKIP":
- functest_utils.push_results_to_db("domino",
- "domino-multinode",
- logger,
- start_time,
- stop_time,
- status,
- details)
+ functest_utils.logger_test_results(logger, "Domino",
+ "domino-multinode",
+ status, details)
+ if args.report:
+ if status is not "SKIP":
+ functest_utils.push_results_to_db("domino",
+ "domino-multinode",
+ logger,
+ start_time,
+ stop_time,
+ status,
+ details)
+ logger.info("Domino results pushed to DB")
+
if __name__ == '__main__':
main()
diff --git a/testcases/features/sfc/set-up-tacker.sh b/testcases/features/sfc/set-up-tacker.sh
index 577b7d94e..da2cb922e 100755
--- a/testcases/features/sfc/set-up-tacker.sh
+++ b/testcases/features/sfc/set-up-tacker.sh
@@ -1,8 +1,8 @@
-curl "https://gerrit.opnfv.org/gerrit/gitweb?p=fuel.git;a=blob_plain;f=prototypes/sfc_tacker/poc.tacker-up.sh;hb=ddd4e11bb8bc62b7e8b06d4b44a308293c2c3362" > poc.tacker-up.sh
+git_commit=ee3046f24df0bfca7ee15501f6c06ad86dd462c2
+curl "https://gerrit.opnfv.org/gerrit/gitweb?p=fuel.git;a=blob_plain;\
+f=prototypes/sfc_tacker/poc.tacker-up.sh;hb=${git_commit}" > poc.tacker-up.sh
bash poc.tacker-up.sh
-touch delete.sh
-
cat <<EOF > delete.sh
tacker sfc-classifier-delete red_http
tacker sfc-classifier-delete blue_ssh
diff --git a/testcases/features/sfc/sfc.py b/testcases/features/sfc/sfc.py
index 17ab386f6..f8af51060 100755
--- a/testcases/features/sfc/sfc.py
+++ b/testcases/features/sfc/sfc.py
@@ -349,6 +349,9 @@ def main():
TACKER_CHANGECLASSI
subprocess.call(tacker_classi, shell=True)
+ logger.info("Wait for ODL to update the classification rules in OVS")
+ time.sleep(10)
+
# SSH to modify the classification flows in compute
contr_cmd4 = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2"
@@ -405,6 +408,21 @@ def main():
logger.info('\033[92m' + "SFC TEST WORKED"
" :) \n" + '\033[0m')
+ # TODO report results to DB
+ # functest_utils.logger_test_results(logger, "SFC",
+ # "odl-sfc",
+ # status, details)
+ # see doctor, promise, domino, ...
+ # if args.report:
+ # logger.info("Pushing odl-SFC results")
+ # functest_utils.push_results_to_db("functest",
+ # "odl-sfc",
+ # logger,
+ # start_time,
+ # stop_time,
+ # status,
+ # details)
+
sys.exit(0)
if __name__ == '__main__':
diff --git a/testcases/security_scan/scripts/internet_check.py b/testcases/security_scan/scripts/internet_check.py
new file mode 100644
index 000000000..1bed50a70
--- /dev/null
+++ b/testcases/security_scan/scripts/internet_check.py
@@ -0,0 +1,25 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Performs simple connection check, falls to default timeout of 10 seconds
+
+import socket
+
+TEST_HOST = "google.com"
+
+
+def is_connected():
+ try:
+ host = socket.gethostbyname(TEST_HOST)
+ socket.create_connection((host, 80), 2)
+ return True
+ except:
+ return False
+print is_connected()
diff --git a/testcases/security_scan/security_scan.py b/testcases/security_scan/security_scan.py
index d39c29052..36b795431 100755
--- a/testcases/security_scan/security_scan.py
+++ b/testcases/security_scan/security_scan.py
@@ -73,21 +73,27 @@ def run_tests(host, nodetype):
port = cfgparse.get(nodetype, 'port')
connect.logger.info("Host: {0} Selected Profile: {1}".format(host,
nodetype))
- connect.logger.info("Creating temp file structure..")
- createfiles(host, port, user, localkey)
- connect.logger.info("Installing OpenSCAP...")
- install_pkg(host, port, user, localkey)
- connect.logger.info("Running scan...")
- run_scanner(host, port, user, localkey, nodetype)
- clean = cfgparse.get(nodetype, 'clean')
- connect.logger.info("Post installation tasks....")
- post_tasks(host, port, user, localkey, nodetype)
- if clean:
- connect.logger.info("Cleaning down environment....")
- connect.logger.info("Removing OpenSCAP....")
- removepkg(host, port, user, localkey, nodetype)
- connect.logger.info("Deleting tmp file and reports (remote)...")
- cleandir(host, port, user, localkey, nodetype)
+ connect.logger.info("Checking internet for package installation...")
+ if internet_check(host, nodetype):
+ connect.logger.info("Internet Connection OK.")
+ connect.logger.info("Creating temp file structure..")
+ createfiles(host, port, user, localkey)
+ connect.logger.info("Installing OpenSCAP...")
+ install_pkg(host, port, user, localkey)
+ connect.logger.info("Running scan...")
+ run_scanner(host, port, user, localkey, nodetype)
+ clean = cfgparse.get(nodetype, 'clean')
+ connect.logger.info("Post installation tasks....")
+ post_tasks(host, port, user, localkey, nodetype)
+ if clean:
+ connect.logger.info("Cleaning down environment....")
+ connect.logger.info("Removing OpenSCAP....")
+ removepkg(host, port, user, localkey, nodetype)
+ connect.logger.info("Deleting tmp file and reports (remote)...")
+ cleandir(host, port, user, localkey, nodetype)
+ else:
+ connect.logger.error("Internet timeout. Moving on to next node..")
+ pass
def nova_iterate():
@@ -106,6 +112,22 @@ def nova_iterate():
run_tests(host, nodetype)
+def internet_check(host, nodetype):
+ import connect
+ user = cfgparse.get(nodetype, 'user')
+ port = cfgparse.get(nodetype, 'port')
+ localpath = functest_dir + 'scripts/internet_check.py'
+ remotepath = '/tmp/internet_check.py'
+ com = 'python /tmp/internet_check.py'
+ testconnect = connect.ConnectionManager(host, port, user, localkey,
+ localpath, remotepath, com)
+ connectionresult = testconnect.remotescript()
+ if connectionresult.rstrip() == 'True':
+ return True
+ else:
+ return False
+
+
def createfiles(host, port, user, localkey):
import connect
global tmpdir
diff --git a/testcases/vnf/vRNC/parser.py b/testcases/vnf/vRNC/parser.py
index 9678591f6..40d28c49f 100644
--- a/testcases/vnf/vRNC/parser.py
+++ b/testcases/vnf/vRNC/parser.py
@@ -44,7 +44,10 @@ def main():
cmd = 'cd %s/tests && ./functest_run.sh' % PARSER_REPO
start_time = time.time()
- ret = functest_utils.execute_command(cmd, logger, exit_on_error=False)
+ ret = functest_utils.execute_command(cmd,
+ logger,
+ info=True,
+ exit_on_error=False)
stop_time = time.time()
status, details = functest_utils.check_test_result(project,
diff --git a/utils/functest_utils.py b/utils/functest_utils.py
index d020ebac8..862b53390 100644
--- a/utils/functest_utils.py
+++ b/utils/functest_utils.py
@@ -169,9 +169,18 @@ def logger_test_results(logger, project, case_name, status, details):
version = get_version(logger)
build_tag = get_build_tag(logger)
- logger.info("Pushing %(p)s/%(n)s results: TEST_DB_URL=%(db)s "
- "pod_name=%(pod)s version=%(v)s scenario=%(s)s "
- "criteria=%(c)s details=%(d)s" % {
+ logger.info("\n"
+ "****************************************\n"
+ "\t %(p)s/%(n)s results \n\n"
+ "****************************************\n"
+ "DB:\t%(db)s\n"
+ "pod:\t%(pod)s\n"
+ "version:\t%(v)s\n"
+ "scenario:\t%(s)s\n"
+ "status:\t%(c)s\n"
+ "build tag:\t%(b)s\n"
+ "details:\t%(d)s\n"
+ % {
'p': project,
'n': case_name,
'db': get_db_url(),
@@ -210,6 +219,7 @@ def push_results_to_db(project, case_name, logger,
r = requests.post(url, data=json.dumps(params), headers=headers)
if logger:
logger.debug(r)
+ r.raise_for_status()
return True
except Exception, e:
print("Error [push_results_to_db('%s', '%s', '%s', " +