diff options
| -rwxr-xr-x | jjb/3rd_party_ci/create-apex-vms.sh | 8 | ||||
| -rwxr-xr-x | jjb/3rd_party_ci/download-netvirt-artifact.sh | 13 | ||||
| -rwxr-xr-x | jjb/3rd_party_ci/functest-netvirt.sh | 4 | ||||
| -rwxr-xr-x | jjb/3rd_party_ci/install-netvirt.sh | 8 | ||||
| -rw-r--r-- | jjb/3rd_party_ci/odl-netvirt.yml | 229 | ||||
| -rwxr-xr-x | jjb/3rd_party_ci/postprocess-netvirt.sh | 4 | ||||
| -rwxr-xr-x | jjb/apex/apex-upload-artifact.sh | 2 | ||||
| -rw-r--r-- | jjb/apex/apex.yml | 2 | ||||
| -rw-r--r-- | jjb/compass4nfv/compass-ci-jobs.yml | 24 | ||||
| -rw-r--r-- | jjb/functest/functest-ci-jobs.yml | 2 | ||||
| -rw-r--r-- | jjb/global/slave-params.yml | 24 | ||||
| -rw-r--r-- | modules/opnfv/utils/ovs_logger.py | 118 |
12 files changed, 424 insertions, 14 deletions
diff --git a/jjb/3rd_party_ci/create-apex-vms.sh b/jjb/3rd_party_ci/create-apex-vms.sh new file mode 100755 index 000000000..cb742712b --- /dev/null +++ b/jjb/3rd_party_ci/create-apex-vms.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# clone opnfv sdnvpn repo +git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git +cd sdnvpn +ls -al + +# here is where the script(s) to bring up apex vms should be executed diff --git a/jjb/3rd_party_ci/download-netvirt-artifact.sh b/jjb/3rd_party_ci/download-netvirt-artifact.sh new file mode 100755 index 000000000..d0b9a05d7 --- /dev/null +++ b/jjb/3rd_party_ci/download-netvirt-artifact.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo "Attempting to fetch the artifact location from ODL Jenkins" +CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/netvirt~master~$GERRIT_CHANGE_ID/detail" +# due to limitation with the Jenkins Gerrit Trigger, we need to use Gerrit REST API to get the change details +ODL_JOB_URL=$(curl -s $CHANGE_DETAILS_URL | grep netvirt-patch-test-current-carbon | tail -1 | \ + sed 's/\\n//g' | awk '{print $6}') +NETVIRT_ARTIFACT_URL="${ODL_JOB_URL}org.opendaylight.integration\$distribution-karaf/artifact/org.opendaylight.integration/distribution-karaf/0.6.0-SNAPSHOT/distribution-karaf-0.6.0-SNAPSHOT.tar.gz" +echo -e "URL to artifact is\n\t$NETVIRT_ARTIFACT_URL" +echo "Downloading the artifact. This could take time..." +curl -s -o $NETVIRT_ARTIFACT $NETVIRT_ARTIFACT_URL +echo "Download complete" +ls -al $NETVIRT_ARTIFACT diff --git a/jjb/3rd_party_ci/functest-netvirt.sh b/jjb/3rd_party_ci/functest-netvirt.sh new file mode 100755 index 000000000..abc7975ce --- /dev/null +++ b/jjb/3rd_party_ci/functest-netvirt.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "Hello World" +ls -al $WORKSPACE diff --git a/jjb/3rd_party_ci/install-netvirt.sh b/jjb/3rd_party_ci/install-netvirt.sh new file mode 100755 index 000000000..041d8517e --- /dev/null +++ b/jjb/3rd_party_ci/install-netvirt.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# clone opnfv sdnvpn repo +git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git +cd sdnvpn +ls -al + +# here is where the script(s) to install netvirt should be executed diff --git a/jjb/3rd_party_ci/odl-netvirt.yml b/jjb/3rd_party_ci/odl-netvirt.yml new file mode 100644 index 000000000..54d1e4353 --- /dev/null +++ b/jjb/3rd_party_ci/odl-netvirt.yml @@ -0,0 +1,229 @@ +- project: + name: 'netvirt' + + project: 'netvirt' + + installer: 'apex' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false +##################################### +# patch verification phases +##################################### + phase: + - 'create-apex-vms': + slave-label: 'ericsson-virtual5' + - 'install-netvirt': + slave-label: 'ericsson-virtual6' + - 'functest': + slave-label: 'ericsson-virtual6' + - 'postprocess': + slave-label: 'ericsson-virtual6' +##################################### +# jobs +##################################### + jobs: + - 'odl-netvirt-verify-virtual-{stream}' + - 'odl-netvirt-verify-virtual-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'odl-netvirt-verify-virtual-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 5 + max-per-node: 1 + option: 'project' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - string: + name: NETVIRT_ARTIFACT + default: $WORKSPACE/distribution-karaf.tar.gz" + - 'ericsson-virtual6-defaults' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + triggers: + - gerrit: + server-name: 'git.opendaylight.org' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : SUCCESS' + - comment-added-contains-event: + comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : UNSTABLE' + - comment-added-contains-event: + comment-contains-value: 'opnfv-test' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + readable-message: true + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: create-apex-vms + condition: SUCCESSFUL + projects: + - name: 'odl-netvirt-verify-virtual-create-apex-vms-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID + GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER + GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: install-netvirt + condition: SUCCESSFUL + projects: + - name: 'odl-netvirt-verify-virtual-install-netvirt-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID + GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER + GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: functest + condition: SUCCESSFUL + projects: + - name: 'odl-netvirt-verify-virtual-functest-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID + GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER + GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: postprocess + condition: SUCCESSFUL + projects: + - name: 'odl-netvirt-verify-virtual-postprocess-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID + GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER + GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'odl-netvirt-verify-virtual-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 5 + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'odl-netvirt-verify-virtual-install-.*' + - 'odl-netvirt-verify-virtual-functest-.*' + - 'odl-netvirt-verify-virtual-postprocess-.*' + block-level: 'NODE' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - '{installer}-defaults' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-verify-{phase}-builder' +##################################### +# builder macros +##################################### +- builder: + name: 'netvirt-verify-create-apex-vms-builder' + builders: + - shell: + !include-raw: ./create-apex-vms.sh +- builder: + name: 'netvirt-verify-install-netvirt-builder' + builders: + - shell: + !include-raw: ./download-netvirt-artifact.sh + - shell: + !include-raw: ./install-netvirt.sh +- builder: + name: 'netvirt-verify-functest-builder' + builders: + - shell: + !include-raw: ./functest-netvirt.sh +- builder: + name: 'netvirt-verify-postprocess-builder' + builders: + - shell: + !include-raw: ./postprocess-netvirt.sh diff --git a/jjb/3rd_party_ci/postprocess-netvirt.sh b/jjb/3rd_party_ci/postprocess-netvirt.sh new file mode 100755 index 000000000..abc7975ce --- /dev/null +++ b/jjb/3rd_party_ci/postprocess-netvirt.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "Hello World" +ls -al $WORKSPACE diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh index f54e4c55a..64f13f4e6 100755 --- a/jjb/apex/apex-upload-artifact.sh +++ b/jjb/apex/apex-upload-artifact.sh @@ -11,6 +11,8 @@ echo # source the opnfv.properties to get ARTIFACT_VERSION source $WORKSPACE/opnfv.properties +BUILD_DIRECTORY=${WORKSPACE}/.build + # clone releng repository echo "Cloning releng repository..." [ -d releng ] && rm -rf releng diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 9befaba83..d8784c678 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -606,7 +606,7 @@ description: "Artifact version type" - string: name: BUILD_DIRECTORY - default: $WORKSPACE/.build + default: $WORKSPACE/build description: "Directory where the build artifact will be located upon the completion of the build." - string: name: CACHE_DIRECTORY diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 330c6dd17..0d97fef75 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -10,10 +10,10 @@ stream: master branch: '{stream}' gs-pathname: '' - colorado: &colorado - stream: colorado - branch: 'stable/{stream}' - gs-pathname: '/{stream}' +# colorado: &colorado +# stream: colorado +# branch: 'stable/{stream}' +# gs-pathname: '/{stream}' #-------------------------------- # POD, INSTALLER, AND BRANCH MAPPING #-------------------------------- @@ -28,14 +28,14 @@ slave-label: compass-virtual os-version: 'trusty' <<: *master - - baremetal: - slave-label: compass-baremetal - os-version: 'trusty' - <<: *colorado - - virtual: - slave-label: compass-virtual - os-version: 'trusty' - <<: *colorado +# - baremetal: +# slave-label: compass-baremetal +# os-version: 'trusty' +# <<: *colorado +# - virtual: +# slave-label: compass-virtual +# os-version: 'trusty' +# <<: *colorado #-------------------------------- # master #-------------------------------- diff --git a/jjb/functest/functest-ci-jobs.yml b/jjb/functest/functest-ci-jobs.yml index 915828ca7..cd7017910 100644 --- a/jjb/functest/functest-ci-jobs.yml +++ b/jjb/functest/functest-ci-jobs.yml @@ -226,7 +226,7 @@ description: 'Tag to pull docker image' - string: name: CLEAN_DOCKER_IMAGES - default: 'true' + default: 'false' description: 'Remove downloaded docker images (opnfv/functest:*)' - functest-parameter: gs-pathname: '{gs-pathname}' diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index dd0c97996..96ad75c69 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -670,6 +670,30 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'ericsson-virtual5-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-virtual5' + - string: + name: GIT_BASE + default: https://git.opendaylight.org/gerrit/p/$PROJECT.git + description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'ericsson-virtual6-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-virtual6' + - string: + name: GIT_BASE + default: https://git.opendaylight.org/gerrit/p/$PROJECT.git + description: 'Git URL to use on this Jenkins Slave' + - string: + name: APEX_ENV_NUMBER + default: 0 + description: 'The number of the Apex environment on the machine' ##################################################### # These slaves are just dummy slaves for sandbox jobs ##################################################### diff --git a/modules/opnfv/utils/ovs_logger.py b/modules/opnfv/utils/ovs_logger.py new file mode 100644 index 000000000..3159609f1 --- /dev/null +++ b/modules/opnfv/utils/ovs_logger.py @@ -0,0 +1,118 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# Author: George Paraskevopoulos (geopar@intracom-telecom.com) +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import opnfv.utils.OPNFVLogger as OPNFVLogger +import os +import time +import shutil + +logger = OPNFVLogger.Logger('ovs_logger').getLogger() + + +class OVSLogger(object): + def __init__(self, basedir, ft_resdir): + self.ovs_dir = basedir + self.ft_resdir = ft_resdir + self.__mkdir_p(self.ovs_dir) + self.__mkdir_p(self.ft_resdir) + + def __mkdir_p(self, dirpath): + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + def __ssh_host(self, ssh_conn, host_prefix='10.20.0'): + try: + _, stdout, _ = ssh_conn.exec_command('hostname -I') + hosts = stdout.readline().strip().split(' ') + found_host = [h for h in hosts if h.startswith(host_prefix)][0] + return found_host + except Exception, e: + logger.error(e) + + def __dump_to_file(self, operation, host, text, timestamp=None): + ts = (timestamp if timestamp is not None + else time.strftime("%Y%m%d-%H%M%S")) + dumpdir = os.path.join(self.ovs_dir, ts) + self.__mkdir_p(dumpdir) + fname = '{0}_{1}'.format(operation, host) + with open(os.path.join(dumpdir, fname), 'w') as f: + f.write(text) + + def __remote_cmd(self, ssh_conn, cmd): + try: + _, stdout, stderr = ssh_conn.exec_command(cmd) + errors = stderr.readlines() + if len(errors) > 0: + host = self.__ssh_host(ssh_conn) + logger.error(''.join(errors)) + raise Exception('Could not execute {0} in {1}' + .format(cmd, host)) + output = ''.join(stdout.readlines()) + return output + except Exception, e: + logger.error('[__remote_command(ssh_client, {0})]: {1}' + .format(cmd, e)) + return None + + def create_artifact_archive(self): + shutil.make_archive(self.ovs_dir, + 'zip', + root_dir=os.path.dirname(self.ovs_dir), + base_dir=self.ovs_dir) + shutil.copy2('{0}.zip'.format(self.ovs_dir), self.ft_resdir) + + def ofctl_dump_flows(self, ssh_conn, br='br-int', + choose_table=None, timestamp=None): + try: + cmd = 'ovs-ofctl -OOpenFlow13 dump-flows {0}'.format(br) + if choose_table is not None: + cmd = '{0} table={1}'.format(cmd, choose_table) + output = self.__remote_cmd(ssh_conn, cmd) + operation = 'ofctl_dump_flows' + host = self.__ssh_host(ssh_conn) + self.__dump_to_file(operation, host, output, timestamp=timestamp) + return output + except Exception, e: + logger.error('[ofctl_dump_flows(ssh_client, {0}, {1})]: {2}' + .format(br, choose_table, e)) + return None + + def vsctl_show(self, ssh_conn, timestamp=None): + try: + cmd = 'ovs-vsctl show' + output = self.__remote_cmd(ssh_conn, cmd) + operation = 'vsctl_show' + host = self.__ssh_host(ssh_conn) + self.__dump_to_file(operation, host, output, timestamp=timestamp) + return output + except Exception, e: + logger.error('[vsctl_show(ssh_client)]: {0}'.format(e)) + return None + + def dump_ovs_logs(self, controller_clients, compute_clients, + related_error=None, timestamp=None): + if timestamp is None: + timestamp = time.strftime("%Y%m%d-%H%M%S") + + for controller_client in controller_clients: + self.ofctl_dump_flows(controller_client, + timestamp=timestamp) + self.vsctl_show(controller_client, + timestamp=timestamp) + + for compute_client in compute_clients: + self.ofctl_dump_flows(compute_client, + timestamp=timestamp) + self.vsctl_show(compute_client, + timestamp=timestamp) + + if related_error is not None: + dumpdir = os.path.join(self.ovs_dir, timestamp) + with open(os.path.join(dumpdir, 'error'), 'w') as f: + f.write(related_error) |