From 3d1f58b82f02881879a53e3187d91e41b5c08081 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Sun, 20 Dec 2015 03:27:09 +0100 Subject: Refactor functest JJB to prepare for Milestone E Replace default stream definitions with stream anchors in order to be able to specify which job on which POD runs against which branch. Move project specific verify job to its own file to ease the stream handling. Change-Id: I293b35f10b93079394026fa75fb71627ee3b04d4 Signed-off-by: Fatih Degirmenci --- jjb/functest/functest-ci-jobs.yml | 292 +++++++++++++++++++++++++++++++++ jjb/functest/functest-project-jobs.yml | 62 +++++++ jjb/functest/functest.yml | 274 ------------------------------- 3 files changed, 354 insertions(+), 274 deletions(-) create mode 100644 jjb/functest/functest-ci-jobs.yml create mode 100644 jjb/functest/functest-project-jobs.yml delete mode 100644 jjb/functest/functest.yml diff --git a/jjb/functest/functest-ci-jobs.yml b/jjb/functest/functest-ci-jobs.yml new file mode 100644 index 000000000..9ae1974a6 --- /dev/null +++ b/jjb/functest/functest-ci-jobs.yml @@ -0,0 +1,292 @@ +################################### +# job configuration for functest +################################### +- project: + name: functest + + project: '{name}' + +#-------------------------------- +# BRANCH ANCHORS +#-------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' + brahmaputra: &brahmaputra + stream: brahmaputra + branch: 'stable/{stream}' + gs-pathname: '{stream}' +#-------------------------------- +# POD, INSTALLER, AND BRANCH MAPPING +#-------------------------------- +# Current Mapping +#-------------------------------- +# everything runs against master branch +#-------------------------------- + pod: + - huawei-us-deploy-bare-1: + installer: compass + <<: *master + - intel-pod5: + installer: joid + <<: *master + - intel-us-deploy-virtual-2: + installer: apex + <<: *master + - opnfv-jump-2: + installer: fuel + <<: *master + - orange-test1: + installer: fuel + <<: *master + - orange-pod2: + installer: joid + <<: *master +#-------------------------------- +# Milestone E Mapping +# !!!DO NOT ENABLE!!! +#-------------------------------- +# brahmaputra +#-------------------------------- +# - huawei-us-deploy-bare-1: +# installer: compass +# <<: *brahmaputra +# - intel-pod5: +# installer: joid +# <<: *brahmaputra +# - opnfv-jump-1: +# installer: apex +# <<: *brahmaputra +# - opnfv-jump-2: +# installer: fuel +# <<: *brahmaputra +#-------------------------------- +# master +#-------------------------------- +# - ericsson-pod2: +# installer: fuel +# <<: *master +# - intelpod2-jumphost: +# installer: apex +# <<: *master +# - intel-pod6: +# installer: joid +# <<: *master +# - intel-pod8: +# installer: compass +# <<: *master +# - orange-test1: +# installer: fuel +# <<: *master +# - orange-pod2: +# installer: joid +# <<: *master +#-------------------------------- + + testsuite: + - 'daily' + - 'suite' + + jobs: + - 'functest-{installer}-{pod}-{testsuite}-{stream}' + +################################ +# job template +################################ +- job-template: + name: 'functest-{installer}-{pod}-{testsuite}-{stream}' + + wrappers: + - build-name: + name: '$BUILD_NUMBER Suite: $FUNCTEST_SUITE_NAME SDN: $SDN_CONTROLLER Feature: $OPNFV_FEATURE' + + parameters: + - project-parameter: + project: '{project}' + - '{pod}-defaults' + - '{installer}-defaults' + - 'functest-{testsuite}-parameter' + - functest-parameter + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + builders: + - 'functest-{testsuite}-builder' + +######################## +# parameter macros +######################## +- parameter: + name: functest-daily-parameter + parameters: + - string: + name: FUNCTEST_SUITE_NAME + default: 'daily' + description: "Suite name to run" +- parameter: + name: functest-suite-parameter + parameters: + - choice: + name: FUNCTEST_SUITE_NAME + choices: + - 'tempest' + - 'rally' + - 'odl' + - 'vims' + - 'vping' +- parameter: + name: functest-parameter + parameters: + - string: + name: GS_PATHNAME + default: '{gs-pathname}' + description: "Version directory where the opnfv documents will be stored in gs repository" + - string: + name: FUNCTEST_REPO_DIR + default: "/home/opnfv/repos/functest" + description: "Directory where the Functest repository is cloned" + - string: + name: PUSH_RESULTS_TO_DB + default: "true" + description: "Push the results of all the tests to the resultDB" +######################## +# trigger macros +######################## +- trigger: + name: 'functest-master' + triggers: + - pollscm: "H 9 * * *" +######################## +# builder macros +######################## +- builder: + name: functest-daily-builder + builders: + - 'functest-cleanup' + - 'set-functest-env' + - 'functest-all' + - 'functest-store-results' + +- builder: + name: functest-suite-builder + builders: + - shell: | + #!/bin/bash + set +e + flag="" + if [ "${PUSH_RESULTS_TO_DB}" == "true" ]; then + flag="-r" + fi + echo "Functest: run $FUNCTEST_SUITE_NAME" + cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME ${flag}" + container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) + docker exec $container_id $cmd + +- builder: + name: functest-all + builders: + - shell: | + #!/bin/bash + set +e + flag="" + if [ "${PUSH_RESULTS_TO_DB}" == "true" ]; then + flag="-r" + fi + echo "Functest: running all the tests" + cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh ${flag}" + container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) + docker exec $container_id $cmd + +- builder: + name: set-functest-env + builders: + - shell: | + #!/bin/bash + set +e + # labconfig is used only for joid + labconfig="" + if [[ ${INSTALLER_TYPE} == 'apex' ]]; then + INSTALLER_IP=$(grep instack /var/lib/libvirt/dnsmasq/default.leases \ + |awk '{print $3}' | head -n 1) + sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" + sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable + sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable + elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then + # If production lab then creds may be retrieved dynamically + # creds are on the jumphost, always in the same folder + labconfig="-v /var/lib/jenkins/joid_config/admin-openrc:/home/opnfv/functest/conf/openstack.creds" + # If dev lab, credentials may not be the default ones, just provide a path to put them into docker + # replace the default one by the customized one provided by jenkins config + if [ -n "${LAB_CONFIG}" ]; then + labconfig="-v ${LAB_CONFIG}:/home/opnfv/functest/conf/openstack.creds" + fi + fi + echo "Functest: Start Docker and prepare environment" + envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME}" + + dir_result="${HOME}/opnfv/functest/reports" + mkdir -f ${dir_result} + rm -rf ${dir_result}/* + $res_volume="-v ${dir_result}:/home/opnfv/functest/results" + + docker pull opnfv/functest:latest_stable + cmd="docker run -id -e $envs ${labconfig} ${sshkey} ${res_volume} opnfv/functest:latest_stable /bin/bash" + echo "Functest: Running docker run command: ${cmd}" + ${cmd} + docker ps -a + sleep 5 + container_id=$(docker ps | grep 'opnfv/functest:latest_stable' | awk '{print $1}' | head -1) + echo "Container ID=${container_id}" + if [ -z ${container_id} ]; then + echo "Cannot find opnfv/functest container ID ${container_id}. Please check if it is existing." + docker ps -a + exit 1 + fi + echo "Starting the container: docker start ${container_id}" + docker start ${container_id} + sleep 5 + docker ps + if [ $(docker ps | grep 'opnfv/functest:latest_stable' | wc -l) == 0 ]; then + echo "The container opnfv/functest with ID=${container_id} has not been properly started. Exiting..." + exit 1 + fi + cmd="${FUNCTEST_REPO_DIR}/docker/prepare_env.sh" + echo "Executing command inside the docker: ${cmd}" + docker exec ${container_id} ${cmd} + +- builder: + name: functest-store-results + builders: + - shell: + !include-raw res-build.sh + +- builder: + name: functest-cleanup + builders: + - shell: | + #!/bin/bash + + echo "Cleaning up docker containers/images..." + + # Remove previous running containers if exist + if [[ ! -z $(docker ps -a | grep opnfv/functest) ]]; then + echo "Removing existing opnfv/functest containers..." + docker ps | grep opnfv/functest | awk '{print $1}' | xargs docker stop + docker ps -a | grep opnfv/functest | awk '{print $1}' | xargs docker rm + fi + + # Remove existing images if exist + if [[ ! -z $(docker images | grep opnfv/functest) ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep opnfv/functest + image_tags=($(docker images | grep opnfv/functest | awk '{print $2}')) + for tag in "${image_tags[@]}"; do + echo "Removing docker image opnfv/functest:$tag..." + docker rmi opnfv/functest:$tag + done + fi diff --git a/jjb/functest/functest-project-jobs.yml b/jjb/functest/functest-project-jobs.yml new file mode 100644 index 000000000..e2a5c1afd --- /dev/null +++ b/jjb/functest/functest-project-jobs.yml @@ -0,0 +1,62 @@ +################################################### +# All the jobs except verify have been removed! +# They will only be enabled on request by projects! +################################################### +- project: + name: functest-project-jobs + + project: 'functest' + + jobs: + - 'functest-verify-{stream}' + +# only master branch is enabled at the moment to keep no of jobs sane + stream: + - master: + branch: 'master' + gs-pathname: '' +# - brahmaputra: +# branch: 'stable/brahmaputra' +# gs-pathname: '/brahmaputra' + +- job-template: + name: 'functest-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - shell: | + echo "Nothing to verify!" diff --git a/jjb/functest/functest.yml b/jjb/functest/functest.yml deleted file mode 100644 index 5aa178bc3..000000000 --- a/jjb/functest/functest.yml +++ /dev/null @@ -1,274 +0,0 @@ -################################### -# job configuration for functest -################################### -- project: - name: functest - - project: '{name}' - - pod: - - intel-us-deploy-virtual-2: - installer: apex - - opnfv-jump-2: - installer: fuel - - orange-test1: - installer: fuel - - orange-pod2: - installer: joid - - huawei-us-deploy-bare-1: - installer: compass - - intel-pod5: - installer: joid - - testsuite: - - 'daily' - - 'suite' - - jobs: - - 'functest-{installer}-{pod}-{testsuite}-{stream}' - - 'functest-verify-{stream}' - - stream: - - master: - branch: 'master' - gs-pathname: '' -# - brahmaputra: -# branch: 'stable/brahmaputra' -# gs-pathname: '/brahmaputra' - -################################ -# job template -################################ -- job-template: - name: 'functest-{installer}-{pod}-{testsuite}-{stream}' - - wrappers: - - build-name: - name: '$BUILD_NUMBER: $FUNCTEST_SUITE_NAME $SDN_CONTROLLER $OPNFV_FEATURE' - - parameters: - - project-parameter: - project: '{project}' - - '{pod}-defaults' - - '{installer}-defaults' - - 'functest-{testsuite}-parameter' - - functest-parameter - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - builders: - - 'functest-{testsuite}-builder' - -- job-template: - name: 'functest-verify-{stream}' - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - triggers: - - gerrit: - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: 'functest' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - - builders: - - shell: | - echo "Nothing to verify!" - -######################## -# parameter macros -######################## -- parameter: - name: functest-daily-parameter - parameters: - - string: - name: FUNCTEST_SUITE_NAME - default: 'daily' - description: "Suite name to run" -- parameter: - name: functest-suite-parameter - parameters: - - choice: - name: FUNCTEST_SUITE_NAME - choices: - - 'tempest' - - 'rally' - - 'odl' - - 'vims' - - 'vping' -- parameter: - name: functest-parameter - parameters: - - string: - name: GS_PATHNAME - default: '{gs-pathname}' - description: "Version directory where the opnfv documents will be stored in gs repository" - - string: - name: FUNCTEST_REPO_DIR - default: "/home/opnfv/repos/functest" - description: "Directory where the Functest repository is cloned" - - string: - name: PUSH_RESULTS_TO_DB - default: "true" - description: "Push the results of all the tests to the resultDB" -######################## -# trigger macros -######################## -- trigger: - name: 'functest-master' - triggers: - - pollscm: "H 9 * * *" -######################## -# builder macros -######################## -- builder: - name: functest-daily-builder - builders: - - 'functest-cleanup' - - 'set-functest-env' - - 'functest-all' - - 'functest-store-results' - -- builder: - name: functest-suite-builder - builders: - - shell: | - #!/bin/bash - set +e - flag="" - if [ "${PUSH_RESULTS_TO_DB}" == "true" ]; then - flag="-r" - fi - echo "Functest: run $FUNCTEST_SUITE_NAME" - cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME ${flag}" - container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) - docker exec $container_id $cmd - -- builder: - name: functest-all - builders: - - shell: | - #!/bin/bash - set +e - flag="" - if [ "${PUSH_RESULTS_TO_DB}" == "true" ]; then - flag="-r" - fi - echo "Functest: running all the tests" - cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh ${flag}" - container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) - docker exec $container_id $cmd - -- builder: - name: set-functest-env - builders: - - shell: | - #!/bin/bash - set +e - # labconfig is used only for joid - labconfig="" - if [[ ${INSTALLER_TYPE} == 'apex' ]]; then - INSTALLER_IP=$(grep instack /var/lib/libvirt/dnsmasq/default.leases \ - |awk '{print $3}' | head -n 1) - sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" - sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable - sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable - elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then - # If production lab then creds may be retrieved dynamically - # creds are on the jumphost, always in the same folder - labconfig="-v /var/lib/jenkins/joid_config/admin-openrc:/home/opnfv/functest/conf/openstack.creds" - # If dev lab, credentials may not be the default ones, just provide a path to put them into docker - # replace the default one by the customized one provided by jenkins config - if [ -n "${LAB_CONFIG}" ]; then - labconfig="-v ${LAB_CONFIG}:/home/opnfv/functest/conf/openstack.creds" - fi - fi - echo "Functest: Start Docker and prepare environment" - envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME}" - - dir_result="${HOME}/opnfv/functest/reports" - mkdir -f ${dir_result} - rm -rf ${dir_result}/* - $res_volume="-v ${dir_result}:/home/opnfv/functest/results" - - docker pull opnfv/functest:latest_stable - cmd="docker run -id -e $envs ${labconfig} ${sshkey} ${res_volume} opnfv/functest:latest_stable /bin/bash" - echo "Functest: Running docker run command: ${cmd}" - ${cmd} - docker ps -a - sleep 5 - container_id=$(docker ps | grep 'opnfv/functest:latest_stable' | awk '{print $1}' | head -1) - echo "Container ID=${container_id}" - if [ -z ${container_id} ]; then - echo "Cannot find opnfv/functest container ID ${container_id}. Please check if it is existing." - docker ps -a - exit 1 - fi - echo "Starting the container: docker start ${container_id}" - docker start ${container_id} - sleep 5 - docker ps - if [ $(docker ps | grep 'opnfv/functest:latest_stable' | wc -l) == 0 ]; then - echo "The container opnfv/functest with ID=${container_id} has not been properly started. Exiting..." - exit 1 - fi - cmd="${FUNCTEST_REPO_DIR}/docker/prepare_env.sh" - echo "Executing command inside the docker: ${cmd}" - docker exec ${container_id} ${cmd} - -- builder: - name: functest-store-results - builders: - - shell: - !include-raw res-build.sh - -- builder: - name: functest-cleanup - builders: - - shell: | - #!/bin/bash - - echo "Cleaning up docker containers/images..." - - # Remove previous running containers if exist - if [[ ! -z $(docker ps -a | grep opnfv/functest) ]]; then - echo "Removing existing opnfv/functest containers..." - docker ps | grep opnfv/functest | awk '{print $1}' | xargs docker stop - docker ps -a | grep opnfv/functest | awk '{print $1}' | xargs docker rm - fi - - # Remove existing images if exist - if [[ ! -z $(docker images | grep opnfv/functest) ]]; then - echo "Docker images to remove:" - docker images | head -1 && docker images | grep opnfv/functest - image_tags=($(docker images | grep opnfv/functest | awk '{print $2}')) - for tag in "${image_tags[@]}"; do - echo "Removing docker image opnfv/functest:$tag..." - docker rmi opnfv/functest:$tag - done - fi -- cgit 1.2.3-korg