summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorxudan <xudan16@huawei.com>2018-07-12 07:00:52 -0400
committerxudan <xudan16@huawei.com>2018-07-17 05:32:46 -0400
commitb9034fde76d6df073043bc8ef6b011c3a0dd5a37 (patch)
tree3dab6fe452593a82183fb93763e2bf09cd3fdc03
parentec7fe48da4ecd10c119d12cc68ea0c2a65cffee9 (diff)
Update test suite ovp.next
Update test suite ovp.next according to the latest OVP scope. Modify Dovetail daily jobs to run all the scope. There should be 2 jobs. One for all mandatory test cases and the other for all optional ones. That's mainly because of that the total executed time will be too large (more than 300 minutes). Then the job will always failed because of time out. It's hard to enlarge the time because it's already larger than common 3 hours. Split it into 2 jobs can avoid the time out issue and make the results clearer for reviewing. JIRA: DOVETAIL-694 Change-Id: Ie0ea6221868941781af1477f7c7719f7cb4351a4 Signed-off-by: xudan <xudan16@huawei.com>
-rw-r--r--dovetail/cli/cli_base.py2
-rw-r--r--dovetail/cli/commands/cli_testcase.py53
-rwxr-xr-xdovetail/run.py9
-rw-r--r--dovetail/testcase.py47
-rw-r--r--etc/compliance/healthcheck.yml7
-rw-r--r--etc/compliance/onap.1.0.0.yml3
-rw-r--r--etc/compliance/ovp.next.yaml70
-rw-r--r--etc/compliance/proposed_tests.yml52
-rw-r--r--etc/conf/cmd_config.yml10
-rw-r--r--etc/conf/dovetail_config.yml11
10 files changed, 151 insertions, 113 deletions
diff --git a/dovetail/cli/cli_base.py b/dovetail/cli/cli_base.py
index c0d57e86..358a5294 100644
--- a/dovetail/cli/cli_base.py
+++ b/dovetail/cli/cli_base.py
@@ -28,7 +28,7 @@ _testcase = CliTestcase()
help='list the testsuite details')
@click.argument('testsuite', type=click.STRING, required=False)
def testcase_list(testsuite):
- _testcase.list_testcase(testsuite)
+ _testcase.list_testsuites(testsuite)
@cli.command('show',
diff --git a/dovetail/cli/commands/cli_testcase.py b/dovetail/cli/commands/cli_testcase.py
index d4d80334..05154882 100644
--- a/dovetail/cli/commands/cli_testcase.py
+++ b/dovetail/cli/commands/cli_testcase.py
@@ -24,39 +24,38 @@ class CliTestcase(object):
dt_cfg.load_config_files(constants.CONF_PATH)
Testsuite.load()
- def list_testcase(self, testsuite):
+ def list_one_testsuite(self, testsuite):
+ testsuite_stream = Testsuite.get(testsuite)
+ if testsuite_stream:
+ mandatory = dt_utils.get_value_from_dict(
+ 'testcases_list.mandatory', testsuite_stream)
+ optional = dt_utils.get_value_from_dict(
+ 'testcases_list.optional', testsuite_stream)
+ if mandatory:
+ click.echo("- mandatory")
+ for testcase in mandatory:
+ click.echo(" {}".format(testcase))
+ if optional:
+ click.echo("- optional")
+ for testcase in optional:
+ click.echo(" {}".format(testcase))
+ if not (mandatory or optional):
+ click.echo("No testcase in testsuite {}".format(testsuite))
+ else:
+ click.echo("testsuite {} does not exist".format(testsuite))
+
+ def list_testsuites(self, testsuite):
self.testsuite_load()
if testsuite:
- testsuite_stream = Testsuite.get(testsuite)
- if testsuite_stream:
- testcase_list = []
- for value in testsuite_stream['testcases_list']:
- if value is not None:
- testcase_list.append(value)
- testarea_list = []
- for testcase in testcase_list:
- testarea = testcase.split('.')[1]
- if testarea not in testarea_list:
- testarea_list.append(testarea)
- for testarea in testarea_list:
- click.echo("- %s" % testarea)
- for testcase in testcase_list:
- if testarea in testcase:
- click.echo(" %s" % testcase)
- else:
- click.echo("testsuite %s does not exist or not supported"
- % testsuite)
+ self.list_one_testsuite(testsuite)
else:
testsuite_json = Testsuite.get_all()
if testsuite_json:
for key, value in testsuite_json.items():
- click.echo("- %s" % key)
- testsuite_stream = Testsuite.get(key)
- if testsuite_stream['testcases_list']:
- for testcase in testsuite_stream['testcases_list']:
- click.echo(" %s" % testcase)
- else:
- click.echo("No testcase in testsuite %s" % key)
+ click.echo("--------------------------")
+ click.echo("Test Suite {}".format(key))
+ click.echo("--------------------------")
+ self.list_one_testsuite(key)
else:
click.echo("No testsuite defined yet in dovetail!!!")
diff --git a/dovetail/run.py b/dovetail/run.py
index b57f9eed..84a448f6 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -198,9 +198,11 @@ def parse_cli(logger=None, **kwargs):
configs = filter_config(kwargs, logger)
if configs is not None:
dt_cfg.update_config(configs)
- dt_cfg.dovetail_config['offline'] = True if kwargs['offline'] else False
- dt_cfg.dovetail_config['noclean'] = True if kwargs['no_clean'] else False
- dt_cfg.dovetail_config['stop'] = True if kwargs['stop'] else False
+ dt_cfg.dovetail_config['offline'] = kwargs['offline']
+ dt_cfg.dovetail_config['noclean'] = kwargs['no_clean']
+ dt_cfg.dovetail_config['stop'] = kwargs['stop']
+ dt_cfg.dovetail_config['mandatory'] = kwargs['mandatory']
+ dt_cfg.dovetail_config['optional'] = kwargs['optional']
if kwargs['no_api_validation']:
dt_cfg.dovetail_config['no_api_validation'] = True
logger.warning('Strict API response validation DISABLED.')
@@ -215,6 +217,7 @@ def check_testcase_list(testcase_list, logger=None):
logger.error('Test case {} is not defined.'.format(tc))
return None
return testcase_list
+ logger.error("There is no test case to be executed.")
return None
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 23220aeb..69b8ee4c 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -253,34 +253,57 @@ class Testcase(object):
if not testarea:
return True, area_full
- mandatory_list = dt_cfg.dovetail_config['mandatory']
- optional_list = dt_cfg.dovetail_config['optional']
for area in testarea:
if area not in dt_cfg.dovetail_config['testarea_supported']:
return False, None
if area == 'full':
return True, area_full
- if area == 'mandatory':
- for mandatory_area in mandatory_list:
- area_no_duplicate.append(mandatory_area)
- continue
- if area == 'optional':
- for optional_area in optional_list:
- area_no_duplicate.append(optional_area)
- continue
area_no_duplicate.append(area)
area_no_duplicate = list(set(area_no_duplicate))
return True, area_no_duplicate
+ @staticmethod
+ def check_testcase_area(testcase, testarea):
+ if not testcase:
+ return False
+ if testarea == 'full' or testarea in testcase:
+ return True
+ else:
+ return False
+
@classmethod
def get_testcase_list(cls, testsuite, testarea):
testcase_list = []
+ selected_tests = []
testcases = dt_utils.get_value_from_dict('testcases_list', testsuite)
+ mandatory = dt_utils.get_value_from_dict('mandatory', testcases)
+ optional = dt_utils.get_value_from_dict('optional', testcases)
if not testcases:
return testcase_list
- for value in testcases:
+ if dt_cfg.dovetail_config['mandatory']:
+ if not mandatory:
+ cls.logger.error("There is no mandatory test case in "
+ "test suite {}".format(testsuite['name']))
+ else:
+ selected_tests += mandatory
+ if dt_cfg.dovetail_config['optional']:
+ if not optional:
+ cls.logger.error("There is no optional test case in "
+ "test suite {}".format(testsuite['name']))
+ else:
+ selected_tests += optional
+ if (not dt_cfg.dovetail_config['mandatory'] and
+ not dt_cfg.dovetail_config['optional']):
+ if mandatory:
+ selected_tests += mandatory
+ if optional:
+ selected_tests += optional
+
+ if not selected_tests:
+ return None
+ for value in selected_tests:
for area in testarea:
- if value is not None and (area == 'full' or area in value):
+ if cls.check_testcase_area(value, area):
testcase_list.append(value)
break
return testcase_list
diff --git a/etc/compliance/healthcheck.yml b/etc/compliance/healthcheck.yml
index 99844917..1db46990 100644
--- a/etc/compliance/healthcheck.yml
+++ b/etc/compliance/healthcheck.yml
@@ -4,6 +4,7 @@
healthcheck:
name: healthcheck
testcases_list:
- - dovetail.healthcheck.tc001
- - dovetail.healthcheck.tc002
- - dovetail.healthcheck.tc003
+ optional:
+ - dovetail.healthcheck.snaps_health_check
+ - dovetail.healthcheck.connection_check
+ - dovetail.healthcheck.api_check
diff --git a/etc/compliance/onap.1.0.0.yml b/etc/compliance/onap.1.0.0.yml
index 9bb4d2f9..43492ab1 100644
--- a/etc/compliance/onap.1.0.0.yml
+++ b/etc/compliance/onap.1.0.0.yml
@@ -2,4 +2,5 @@
onap.1.0.0:
name: onap.1.0.0
testcases_list:
- - dovetail.lifecycle.tc001 \ No newline at end of file
+ optional:
+ - dovetail.lifecycle.tc001
diff --git a/etc/compliance/ovp.next.yaml b/etc/compliance/ovp.next.yaml
index 67c5d743..7693b921 100644
--- a/etc/compliance/ovp.next.yaml
+++ b/etc/compliance/ovp.next.yaml
@@ -2,33 +2,43 @@
ovp.next:
name: ovp.next
testcases_list:
- # mandatory test cases
- # osinterop
- - dovetail.tempest.osinterop
- # vping
- - dovetail.vping.userdata
- - dovetail.vping.ssh
- # HA
- - dovetail.ha.nova_api
- - dovetail.ha.neutron_server
- - dovetail.ha.keystone
- - dovetail.ha.glance_api
- - dovetail.ha.cinder_api
- - dovetail.ha.cpu_load
- - dovetail.ha.disk_load
- - dovetail.ha.haproxy
- # optional test cases
- # ipv6
- - dovetail.tempest.ipv6_api
- - dovetail.tempest.ipv6_scenario
- # tempest
- # some of the previous tempest test suites tc001 - tc005 have been merged
- # with the smoke tests executed as part of the proposed_test test suite
- - dovetail.tempest.multi_node_scheduling
- - dovetail.tempest.network_security
- - dovetail.tempest.vm_lifecycle
- # sdnvpn
- - dovetail.sdnvpn.subnet_connectivity
- - dovetail.sdnvpn.tenant_separation
- - dovetail.sdnvpn.router_association
- - dovetail.sdnvpn.router_association_floating_ip
+ mandatory:
+ - dovetail.vping.userdata
+ - dovetail.vping.ssh
+ - dovetail.tempest.osinterop
+ - dovetail.tempest.compute
+ - dovetail.tempest.identity_v2
+ - dovetail.tempest.identity_v3
+ - dovetail.tempest.image
+ - dovetail.tempest.network_api
+ - dovetail.tempest.volume
+ - dovetail.tempest.neutron_trunk_ports
+ - dovetail.tempest.ipv6_api
+ - dovetail.security.patrole
+ - dovetail.ha.nova_api
+ - dovetail.ha.neutron_server
+ - dovetail.ha.keystone
+ - dovetail.ha.glance_api
+ - dovetail.ha.cinder_api
+ - dovetail.ha.cpu_load
+ - dovetail.ha.disk_load
+ - dovetail.ha.haproxy
+ - dovetail.ha.rabbitmq
+ - dovetail.ha.database
+ - dovetail.stress.ping
+ optional:
+ - dovetail.tempest.ipv6_scenario
+ - dovetail.tempest.multi_node_scheduling
+ - dovetail.tempest.network_security
+ - dovetail.tempest.vm_lifecycle
+ - dovetail.tempest.network_scenario
+ - dovetail.tempest.bgpvpn
+ - dovetail.sdnvpn.subnet_connectivity
+ - dovetail.sdnvpn.tenant_separation
+ - dovetail.sdnvpn.router_association
+ - dovetail.sdnvpn.router_association_floating_ip
+ - dovetail.ha.neutron_l3_agent
+ - dovetail.ha.controller_restart
+ - dovetail.vnf.vims
+ - dovetail.vnf.vepc
+ - dovetail.snaps.smoke
diff --git a/etc/compliance/proposed_tests.yml b/etc/compliance/proposed_tests.yml
index 6ca82daa..e893fa19 100644
--- a/etc/compliance/proposed_tests.yml
+++ b/etc/compliance/proposed_tests.yml
@@ -3,28 +3,30 @@ proposed_tests:
name: proposed_tests
testcases_list:
# proposed test cases for 2nd release
- # tempest
- - dovetail.tempest.compute
- - dovetail.tempest.identity_v2
- - dovetail.tempest.identity_v3
- - dovetail.tempest.image
- - dovetail.tempest.network_api
- - dovetail.tempest.network_scenario
- - dovetail.tempest.volume
- - dovetail.tempest.neutron_trunk_ports
- # HA
- - dovetail.ha.controller_restart
- - dovetail.ha.rabbitmq
- - dovetail.ha.neutron_l3_agent
- - dovetail.ha.database
- # vnf
- - dovetail.vnf.vims
- - dovetail.vnf.vepc
- # security
- - dovetail.security.patrole
- # sdnvpn
- - dovetail.tempest.bgpvpn
- # snaps
- - dovetail.snaps.smoke
- # stress
- - dovetail.stress.ping
+ mandatory:
+ # tempest
+ - dovetail.tempest.compute
+ - dovetail.tempest.identity_v2
+ - dovetail.tempest.identity_v3
+ - dovetail.tempest.image
+ - dovetail.tempest.network_api
+ - dovetail.tempest.volume
+ - dovetail.tempest.neutron_trunk_ports
+ # HA
+ - dovetail.ha.rabbitmq
+ - dovetail.ha.database
+ # security
+ - dovetail.security.patrole
+ # stress
+ - dovetail.stress.ping
+ optional:
+ - dovetail.tempest.network_scenario
+ - dovetail.ha.controller_restart
+ - dovetail.ha.neutron_l3_agent
+ # vnf
+ - dovetail.vnf.vims
+ - dovetail.vnf.vepc
+ # sdnvpn
+ - dovetail.tempest.bgpvpn
+ # snaps
+ - dovetail.snaps.smoke
diff --git a/etc/conf/cmd_config.yml b/etc/conf/cmd_config.yml
index 76a4d61c..afd06425 100644
--- a/etc/conf/cmd_config.yml
+++ b/etc/conf/cmd_config.yml
@@ -69,3 +69,13 @@ cli:
flags:
- '--deploy-scenario'
help: 'Specify the DEPLOY_SCENARIO which will be used as input by each testcase respectively'
+ mandatory:
+ flags:
+ - '--mandatory'
+ is_flag: 'True'
+ help: 'Run all mandatory test cases.'
+ optional:
+ flags:
+ - '--optional'
+ is_flag: 'True'
+ help: 'Run all optional test cases.'
diff --git a/etc/conf/dovetail_config.yml b/etc/conf/dovetail_config.yml
index 83944459..0150e2d8 100644
--- a/etc/conf/dovetail_config.yml
+++ b/etc/conf/dovetail_config.yml
@@ -29,7 +29,6 @@ trusty_image: 'trusty-server-cloudimg-amd64-disk1.img'
# testsuite supported, should adjust accordingly
testsuite_supported:
- - compliance_set
- proposed_tests
- debug
- healthcheck
@@ -90,13 +89,3 @@ test_project:
- 'functest'
- 'bottlenecks'
- 'vnftest'
-
-mandatory:
- - osinterop
- - ha
- - vping
-
-optional:
- - ipv6
- - tempest
- - sdnvpn