summaryrefslogtreecommitdiffstats
path: root/xtesting
diff options
context:
space:
mode:
Diffstat (limited to 'xtesting')
-rw-r--r--xtesting/ansible/host_vars/127.0.0.13
-rw-r--r--xtesting/ansible/site.yml4
-rw-r--r--xtesting/behaveframework.py123
-rw-r--r--xtesting/testcases.yaml62
4 files changed, 190 insertions, 2 deletions
diff --git a/xtesting/ansible/host_vars/127.0.0.1 b/xtesting/ansible/host_vars/127.0.0.1
index 9e3f1b9..125032f 100644
--- a/xtesting/ansible/host_vars/127.0.0.1
+++ b/xtesting/ansible/host_vars/127.0.0.1
@@ -1,5 +1,8 @@
docker_args:
env: {}
+ params:
+ net: host
+ privileged: true
volumes:
- /lib/modules/$(uname -r):/lib/modules/$(uname -r)
- /usr/src/kernels:/usr/src/kernels -v /dev:/dev
diff --git a/xtesting/ansible/site.yml b/xtesting/ansible/site.yml
index 4643a32..37fa6c3 100644
--- a/xtesting/ansible/site.yml
+++ b/xtesting/ansible/site.yml
@@ -16,5 +16,7 @@
- container: nfvbench
tests:
- 10kpps-pvp-run
+ - characterization
+ - non-regression
properties:
- execution-type: SEQUENTIALLY
+ execution-type: SEQUENTIALLY \ No newline at end of file
diff --git a/xtesting/behaveframework.py b/xtesting/behaveframework.py
new file mode 100644
index 0000000..651240d
--- /dev/null
+++ b/xtesting/behaveframework.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# Copyright 2021 Orange
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+"""Define classes required to run any Behave test suites."""
+
+from __future__ import division
+
+import logging
+import os
+import time
+
+import json
+import six
+
+from behave.__main__ import main as behave_main
+
+from xtesting.core import testcase
+
+__author__ = "Deepak Chandella <deepak.chandella@orange.com>"
+
+
+class BehaveFramework(testcase.TestCase):
+ """BehaveFramework runner."""
+ # pylint: disable=too-many-instance-attributes
+
+ __logger = logging.getLogger(__name__)
+ dir_results = "/var/lib/xtesting/results"
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.json_file = os.path.join(self.res_dir, 'output.json')
+ self.total_tests = 0
+ self.pass_tests = 0
+ self.fail_tests = 0
+ self.skip_tests = 0
+ self.response = None
+
+ def parse_results(self):
+ """Parse output.json and get the details in it."""
+ with open(self.json_file) as stream_:
+ self.response = json.load(stream_)
+ if self.response:
+ self.total_tests = len(self.response)
+ for item in self.response:
+ if item['status'] == 'passed':
+ self.pass_tests += 1
+ elif item['status'] == 'failed':
+ self.fail_tests += 1
+ elif item['status'] == 'skipped':
+ self.skip_tests += 1
+ self.result = 100 * (
+ self.pass_tests / self.total_tests)
+ self.details = {}
+ self.details['total_tests'] = self.total_tests
+ self.details['pass_tests'] = self.pass_tests
+ self.details['fail_tests'] = self.fail_tests
+ self.details['skip_tests'] = self.skip_tests
+ self.details['tests'] = self.response
+
+ def run(self, **kwargs):
+ """Run the BehaveFramework feature files
+
+ Here are the steps:
+ * create the output directories if required,
+ * run behave features with parameters
+ * get the results in output.json,
+
+ Args:
+ kwargs: Arbitrary keyword arguments.
+
+ Returns:
+ EX_OK if all suites ran well.
+ EX_RUN_ERROR otherwise.
+ """
+ try:
+ suites = kwargs["suites"]
+ tags = kwargs.get("tags", [])
+ console = kwargs["console"] if "console" in kwargs else False
+ except KeyError:
+ self.__logger.exception("Mandatory args were not passed")
+ return self.EX_RUN_ERROR
+ if not os.path.exists(self.res_dir):
+ try:
+ os.makedirs(self.res_dir)
+ except Exception: # pylint: disable=broad-except
+ self.__logger.exception("Cannot create %s", self.res_dir)
+ return self.EX_RUN_ERROR
+ config = ['--tags=' + ','.join(tags),
+ '--junit', '--junit-directory={}'.format(self.res_dir),
+ '--format=json', '--outfile={}'.format(self.json_file)]
+ if six.PY3:
+ html_file = os.path.join(self.res_dir, 'output.html')
+ config += ['--format=behave_html_formatter:HTMLFormatter',
+ '--outfile={}'.format(html_file)]
+ if console:
+ config += ['--format=pretty',
+ '--outfile=-']
+ for feature in suites:
+ config.append(feature)
+ self.start_time = time.time()
+ behave_main(config)
+ self.stop_time = time.time()
+
+ try:
+ self.parse_results()
+ self.__logger.info("Results were successfully parsed")
+ except Exception: # pylint: disable=broad-except
+ self.__logger.exception("Cannot parse results")
+ return self.EX_RUN_ERROR
+ return self.EX_OK
diff --git a/xtesting/testcases.yaml b/xtesting/testcases.yaml
index cbb5c45..eea60b9 100644
--- a/xtesting/testcases.yaml
+++ b/xtesting/testcases.yaml
@@ -6,6 +6,65 @@ tiers:
description: 'Data Plane Performance Testing'
testcases:
-
+ case_name: characterization
+ project_name: nfvbench
+ criteria: 100
+ blocking: true
+ clean_flag: false
+ description: ''
+ run:
+ name: 'nfvbench_behaveframework'
+ args:
+ suites:
+ - /opt/nfvbench/behave_tests/features/characterization-full.feature
+ tags:
+ - characterization
+ console:
+ - true
+ -
+ case_name: non-regression
+ project_name: nfvbench
+ criteria: 100
+ blocking: true
+ clean_flag: false
+ description: ''
+ run:
+ name: 'nfvbench_behaveframework'
+ args:
+ suites:
+ - /opt/nfvbench/behave_tests/features/non-regression.feature
+ tags:
+ - non-regression
+ console:
+ - true
+
+ -
+ name: nfvbench-rapid-characterization
+ order: 2
+ description: 'Data Plane Performance Testing'
+ testcases:
+ -
+ case_name: rapid-characterization
+ project_name: nfvbench
+ criteria: 100
+ blocking: true
+ clean_flag: false
+ description: ''
+ run:
+ name: 'nfvbench_behaveframework'
+ args:
+ suites:
+ - /opt/nfvbench/behave_tests/features/characterization-samples.feature
+ tags:
+ - characterization
+ console:
+ - true
+ -
+ name: nfvbench-demo
+ order: 3
+ description: 'Data Plane Performance Testing'
+ testcases:
+ -
case_name: 10kpps-pvp-run
project_name: nfvbench
criteria: 100
@@ -17,4 +76,5 @@ tiers:
args:
cmd:
- nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps
-
+ console:
+ - true \ No newline at end of file