diff options
| -rwxr-xr-x | jjb/apex/apex-build.sh | 1 | ||||
| -rwxr-xr-x | jjb/apex/apex-deploy.sh | 6 | ||||
| -rw-r--r-- | jjb/armband/armband-ci-jobs.yml | 2 | ||||
| -rw-r--r-- | jjb/functest/functest-daily-jobs.yml | 48 | ||||
| -rw-r--r-- | jjb/releng/automate.yml | 2 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker.yml | 4 | ||||
| -rw-r--r-- | jjb/xci/xci-verify-jobs.yml | 7 | ||||
| -rw-r--r-- | modules/opnfv/deployment/OSA/__init__.py | 0 | ||||
| -rw-r--r-- | modules/opnfv/deployment/OSA/adapter.py | 76 | ||||
| -rw-r--r-- | modules/opnfv/deployment/factory.py | 7 | ||||
| -rw-r--r-- | utils/test/testapi/.gitignore | 1 | ||||
| -rw-r--r-- | utils/test/testapi/README.rst | 101 | ||||
| -rw-r--r-- | utils/test/testapi/opnfv_testapi/common/check.py | 8 | ||||
| -rw-r--r-- | utils/test/testapi/opnfv_testapi/tests/unit/fake_pymongo.py | 12 | ||||
| -rw-r--r-- | utils/test/testapi/opnfv_testapi/tests/unit/resources/test_pod.py | 7 |
15 files changed, 224 insertions, 58 deletions
diff --git a/jjb/apex/apex-build.sh b/jjb/apex/apex-build.sh index ad94ba3d7..23ce81015 100755 --- a/jjb/apex/apex-build.sh +++ b/jjb/apex/apex-build.sh @@ -31,6 +31,7 @@ BUILD_DIRECTORY=${WORKSPACE}/build pushd ${BUILD_DIRECTORY} make clean popd +export PYTHONPATH=${WORKSPACE} python3 apex/build.py $BUILD_ARGS RPM_VERSION=$(grep Version: $WORKSPACE/build/rpm_specs/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-') # list the contents of BUILD_OUTPUT directory diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index a47e3a5db..bfed67ea5 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -49,10 +49,10 @@ if [[ "$ARTIFACT_VERSION" =~ dev ]]; then # TODO(trozet) add verification here # Install dev build - mkdir -p ~/tmp - mv -f .build ~/tmp/ + sudo rm -rf /tmp/.build + mv -f .build /tmp/ sudo pip3 install --upgrade --force-reinstall . - mv -f ~/tmp/.build . + mv -f /tmp/.build ${WORKSPACE}/ else DEPLOY_SETTINGS_DIR="/etc/opnfv-apex/" NETWORK_SETTINGS_DIR="/etc/opnfv-apex/" diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index faa5971e1..276f68809 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -113,7 +113,7 @@ same-node: true block: true - trigger-builds: - - project: 'functest-{installer}-{pod}-daily-{stream}' + - project: 'functest-{installer}-{pod}-arm-daily-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index b1d7e748d..cbf175567 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -192,6 +192,8 @@ job-timeout: 60 - 'daily': job-timeout: 240 + - 'arm-daily': + job-timeout: 240 jobs: - 'functest-{installer}-{pod}-{testsuite}-{stream}' @@ -299,12 +301,21 @@ default: 'daily' description: "Daily suite name to run" - parameter: + name: functest-arm-daily-parameter + parameters: + - string: + name: FUNCTEST_SUITE_NAME + default: 'arm-daily' + description: "Daily suite name (Aarch64) to run" +- parameter: name: functest-suite-parameter parameters: - choice: name: FUNCTEST_SUITE_NAME choices: - - 'healthcheck' + - 'connection_check' + - 'api_check' + - 'snaps_health_check' - 'vping_userdata' - 'vping_ssh' - 'tempest_smoke_serial' @@ -312,16 +323,19 @@ - 'odl' - 'odl_netvirt' - 'onos' + - 'snaps_smoke' + - 'refstack_defcore' - 'promise' - 'doctor' - 'bgpvpn' + - 'parser' - 'security_scan' - 'tempest_full_parallel' - 'rally_full' - - 'vims' - - 'multisite' - - 'parser' - - 'opera_vims' + - 'cloudify_ims' + - 'cloudify_vrouter' + - 'orchestra_openims' + - 'orchestra_clearwaterims' - string: name: TESTCASE_OPTIONS default: '' @@ -369,6 +383,15 @@ - 'functest-store-results' - builder: + name: functest-arm-daily-builder + builders: + - 'functest-cleanup' + - 'set-functest-env' + - 'functest-arm-daily' + - 'functest-store-results' + - 'functest-exit' + +- builder: name: functest-suite-builder builders: - 'functest-cleanup' @@ -380,12 +403,10 @@ - builder: name: functest-alpine-daily-builder builders: - - shell: - !include-raw: - - ./functest-env-presetup.sh - - ../../utils/fetch_os_creds.sh - - ./functest-alpine.sh - - ../../utils/push-test-logs.sh + - 'functest-cleanup' + - 'set-functest-env-alpine' + - 'functest-daily' + - 'functest-store-results' - builder: name: functest-daily @@ -393,6 +414,11 @@ - shell: !include-raw: ./functest-alpine.sh +- builder: + name: functest-arm-daily + builders: + - shell: + !include-raw: ./functest-loop.sh - builder: name: functest-suite diff --git a/jjb/releng/automate.yml b/jjb/releng/automate.yml index c6ca37fa9..908e9a196 100644 --- a/jjb/releng/automate.yml +++ b/jjb/releng/automate.yml @@ -247,7 +247,7 @@ builders: - shell: | sudo bash ./jjb/releng/docker-deploy.sh "sudo docker run -itd --name reporting -p 8084:8000 opnfv/reporting" \ - "http://testresults.opnfv.org/reporting2/reporting/index.html" "reporting" + "http://testresults.opnfv.org/reporting/index.html" "reporting" - builder: name: mongodb-backup diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 414eba255..5b63f3a12 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -51,6 +51,10 @@ project: 'functest' <<: *master <<: *functest-receivers + - 'nfvbench': + project: 'nfvbench' + <<: *master + <<: *other-receivers - 'qtip': project: 'qtip' <<: *master diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 2cdecb208..61dda942a 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -87,7 +87,10 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'false' + disable-strict-forbidden-file-verification: 'true' + file-paths: + - compare-type: ANT + pattern: 'xci/**' forbidden-file-paths: - compare-type: ANT pattern: 'bifrost/**' @@ -97,6 +100,8 @@ pattern: 'upstream/**' - compare-type: ANT pattern: '**/README.rst' + - compare-type: ANT + pattern: 'docs/**' readable-message: true parameters: diff --git a/modules/opnfv/deployment/OSA/__init__.py b/modules/opnfv/deployment/OSA/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/modules/opnfv/deployment/OSA/__init__.py diff --git a/modules/opnfv/deployment/OSA/adapter.py b/modules/opnfv/deployment/OSA/adapter.py new file mode 100644 index 000000000..e98526cad --- /dev/null +++ b/modules/opnfv/deployment/OSA/adapter.py @@ -0,0 +1,76 @@ +############################################################################## +# Copyright (c) 2017 SUSE Linux GmbH +# Author: Manuel Buil (mbuil@suse.com) +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from opnfv.deployment import manager +from opnfv.utils import opnfv_logger as logger +from opnfv.utils import ssh_utils +import yaml + +logger = logger.Logger(__name__).getLogger() + + +class OSAAdapter(manager.DeploymentHandler): + + def __init__(self, installer_ip, installer_user, pkey_file): + self.SOURCE_PATH_UC = "/etc/openstack_deploy/openstack_user_config.yml" + self.DEST_PATH_UC = "/tmp/openstack_user_config.yml" + super(OSAAdapter, self).__init__(installer='osa', + installer_ip=installer_ip, + installer_user=installer_user, + installer_pwd=None, + pkey_file=pkey_file) + + def _find_nodes(self, file): + nodes = file['compute_hosts'] + controllers = file['haproxy_hosts'] + for controller in controllers: + nodes[controller] = controllers[controller] + return nodes + + def _process_nodes(self, raw_nodes): + nodes = [] + + for node in raw_nodes: + name = node + ip = raw_nodes[node]['ip'] + # TODO when xci provides status and id of nodes add logic + status = 'active' + id = None + if 'controller' in node: + roles = 'controller' + elif 'compute' in node: + roles = 'compute' + ssh_client = ssh_utils.get_ssh_client(hostname=ip, + username=self.installer_user, + pkey_file=self.pkey_file) + node = manager.Node(id, ip, name, status, roles, ssh_client) + nodes.append(node) + + return nodes + + def get_nodes(self, options=None): + try: + # if we have retrieved previously all the nodes, don't do it again + # This fails the first time when the constructor calls this method + # therefore the try/except + if len(self.nodes) > 0: + return self.nodes + except: + pass + + self.installer_node.get_file(self.SOURCE_PATH_UC, self.DEST_PATH_UC) + with open(self.DEST_PATH_UC, 'r') as stream: + try: + file = yaml.load(stream) + raw_nodes = self._find_nodes(file) + except yaml.YAMLError as exc: + logger.error(exc) + nodes = self._process_nodes(raw_nodes) + return nodes diff --git a/modules/opnfv/deployment/factory.py b/modules/opnfv/deployment/factory.py index b8e5c8ef4..55fb2e2e3 100644 --- a/modules/opnfv/deployment/factory.py +++ b/modules/opnfv/deployment/factory.py @@ -11,6 +11,7 @@ from opnfv.deployment.apex import adapter as apex_adapter from opnfv.deployment.compass import adapter as compass_adapter from opnfv.deployment.fuel import adapter as fuel_adapter +from opnfv.deployment.OSA import adapter as osa_adapter from opnfv.utils import opnfv_logger as logger logger = logger.Logger(__name__).getLogger() @@ -18,7 +19,7 @@ logger = logger.Logger(__name__).getLogger() class Factory(object): - INSTALLERS = ["fuel", "apex", "compass", "joid", "daisy"] + INSTALLERS = ["fuel", "apex", "compass", "joid", "daisy", "OSA"] def __init__(self): pass @@ -46,6 +47,10 @@ class Factory(object): installer_ip=installer_ip, installer_user=installer_user, installer_pwd=installer_pwd) + elif installer.lower() == "osa": + return osa_adapter.OSAAdapter(installer_ip=installer_ip, + installer_user=installer_user, + pkey_file=pkey_file) else: raise Exception("Installer adapter is not implemented for " "the given installer.") diff --git a/utils/test/testapi/.gitignore b/utils/test/testapi/.gitignore index 86ec0d2d5..a3d6e01ec 100644 --- a/utils/test/testapi/.gitignore +++ b/utils/test/testapi/.gitignore @@ -5,3 +5,4 @@ opnfv_testapi/static build *.egg-info 3rd_party/static/static +*.pyc diff --git a/utils/test/testapi/README.rst b/utils/test/testapi/README.rst index 44ab2a475..0d18b7e93 100644 --- a/utils/test/testapi/README.rst +++ b/utils/test/testapi/README.rst @@ -2,63 +2,92 @@ opnfv-testapi ============= -Test Results Collector of OPNFV Test Projects +**Test Results Collector of OPNFV Test Projects**: -Start Server -============== +This project aims to provide a common way of gathering all the test results for OPNFV +testing projects into a single place, and a unified way of getting those results for +testing analysis projects, such as Reporting/Bitergia Dashboard/Qtip. It exposes +Restful APIs for collecting results and pushing them into a MongoDB database. + +If you are interested in how TestAPI looks like, please visit OPNFV's official `TestAPI Server`__ + +.. __: http://testresults.opnfv.org/test + +Pre-requsites +============= -Getting setup -^^^^^^^^^^^^^ +TestAPI leverages MongoDB as the data warehouse, in order to let it work +successfully, a MongoDB must be provided, the official MongoDB version in OPNFV +TestAPI is 3.2.6. And the database is **test_results_collection**, the five +collections are *users/pods/projects/testcases/scenarioes/results*. And thanks to +MongoDB's very user-friendly property, they don't need to be created in advance. -Requirements for opnfv-testapi: +Running locally +=============== -* tornado -* epydoc +Requirements +^^^^^^^^^^^^ -These requirements are expressed in the requirements.txt file and may be -installed by running the following (from within a virtual environment):: +All requirements are listed in requirements.txt and should be +installed by 'pip install': - pip install -r requirements.txt + *pip install -r requirements.txt* + +Installation +^^^^^^^^^^^^ + + *python setup.py install* + +After installation, configuration file etc/config.ini will be put under +/etc/opnfv_testapi/. And all the web relevant files under 3rd_party/static +will be in /user/local/share/opnfv_testapi as a totality. + +Start Server +^^^^^^^^^^^^ -How to install -^^^^^^^^^^^^^^ + *opnfv-testapi [--config-file <config.ini>]* -From within your environment, just run: +If --config-file is provided, the specified configuration file will be employed +when starting the server, or else /etc/opnfv_testapi/config.ini will be utilized +by default. - ./install.sh +After executing the command successfully, a TestAPI server will be started on +port 8000, to visit web portal, please access http://hostname:8000 -How to run -^^^^^^^^^^ +Regarding swagger-ui website, please visit http://hostname:8000/swagger/spec.html -From within your environment, just run: +Running with container +====================== - opnfv-testapi +TestAPI has already containerized, the docker image is opnfv/testapi:latest. -This will start a server on port 8000. Just visit http://localhost:8000 +**Running the container not behind nginx:** -For swagger website, just visit http://localhost:8000/swagger/spec.html +.. code-block:: shell -Unittest -===================== + docker run -dti --name testapi -p expose_port:8000 + -e "mongodb_url=mongodb://mongodb_ip:27017/" + -e "base_url=http://host_name:expose_port" + opnfv/testapi:latest -Getting setup -^^^^^^^^^^^^^ +**Running the container behind nginx:** -Requirements for unittest: +.. code-block:: shell -* testtools -* discover -* futures + docker run -dti --name testapi -p expose_port:8000 + -e "mongodb_url=mongodb://mongodb_ip:27017/" + -e "base_url=http://nginx_url" + opnfv/testapi:latest -These requirements are expressed in the test-requirements.txt file and may be -installed by running the following (from within a virtual environment):: +Unittest & pep8 +=============== - pip install -r test-requirements.txt +All requirements for unit tests are outlined in test-requirements.txt, and in TestAPI project, tox is leveraged to drive the executing of unit tests and pep8 check -How to run -^^^^^^^^^^ +**Running unit tests** -From within your environment, just run:: + *tox -e py27* - bash run_test.sh +**Running pep8 check** + *tox -e pep8* diff --git a/utils/test/testapi/opnfv_testapi/common/check.py b/utils/test/testapi/opnfv_testapi/common/check.py index 24ba876a9..acd331784 100644 --- a/utils/test/testapi/opnfv_testapi/common/check.py +++ b/utils/test/testapi/opnfv_testapi/common/check.py @@ -7,6 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import functools +import re from tornado import gen from tornado import web @@ -92,7 +93,12 @@ def new_not_exists(xstep): def wrap(self, *args, **kwargs): query = kwargs.get('query') if query: - to_data = yield dbapi.db_find_one(self.table, query()) + query_data = query() + if self.table == 'pods': + if query_data.get('name') is not None: + query_data['name'] = re.compile(query_data.get('name'), + re.IGNORECASE) + to_data = yield dbapi.db_find_one(self.table, query_data) if to_data: raises.Forbidden(message.exist(self.table, query())) ret = yield gen.coroutine(xstep)(self, *args, **kwargs) diff --git a/utils/test/testapi/opnfv_testapi/tests/unit/fake_pymongo.py b/utils/test/testapi/opnfv_testapi/tests/unit/fake_pymongo.py index 0ca83df62..3320a866a 100644 --- a/utils/test/testapi/opnfv_testapi/tests/unit/fake_pymongo.py +++ b/utils/test/testapi/opnfv_testapi/tests/unit/fake_pymongo.py @@ -6,6 +6,8 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import re + from operator import itemgetter from bson.objectid import ObjectId @@ -190,8 +192,13 @@ class MemDb(object): elif k == 'trust_indicator.current': if content.get('trust_indicator').get('current') != v: return False - elif not isinstance(v, dict) and content.get(k, None) != v: - return False + elif not isinstance(v, dict): + if isinstance(v, re._pattern_type): + if v.match(content.get(k, None)) is None: + return False + else: + if content.get(k, None) != v: + return False return True def _find(self, *args): @@ -199,7 +206,6 @@ class MemDb(object): for content in self.contents: if self._in(content, *args): res.append(content) - return res def find(self, *args): diff --git a/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_pod.py b/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_pod.py index cb4f1d92c..d1a19f7f0 100644 --- a/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_pod.py +++ b/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_pod.py @@ -21,6 +21,8 @@ class TestPodBase(base.TestBase): self.req_d = pod_models.PodCreateRequest('zte-1', 'virtual', 'zte pod 1', 'ci-pod') self.req_e = pod_models.PodCreateRequest('zte-2', 'metal', 'zte pod 2') + self.req_f = pod_models.PodCreateRequest('Zte-1', 'virtual', + 'zte pod 1', 'ci-pod') self.get_res = pod_models.Pod self.list_res = pod_models.Pods self.basePath = '/api/v1/pods' @@ -58,6 +60,11 @@ class TestPodCreate(TestPodBase): self.create_d() return self.req_d + @executor.create(httplib.FORBIDDEN, message.exist_base) + def test_alreadyExistCaseInsensitive(self): + self.create(self.req_f) + return self.req_d + class TestPodGet(TestPodBase): def setUp(self): |