From 0466d549e60cb77068242a2df1bac0cbe262975a Mon Sep 17 00:00:00 2001 From: rohitsakala Date: Thu, 29 Jun 2017 22:08:05 +0530 Subject: Add jenkins jobs for automatic reporting dashboard deployment Jenkins Jobs have been added for automating the deployment of the reporting dashboard. Also multijob has been added for automating the update of docker in the docker hub and generating docs which is empty for now. Modified unit tests of reporting to make it run from any directory. Modified the testapi jenkins jobs yaml file to generic so that many other modules which may have the same usecases can be integrated easily just like the reporting one that is done now. Change-Id: Id13a55416e17199dae1dd6ce64f7c167d117ba37 Signed-off-by: Sakala Venkata Krishna Rohit --- jjb/releng/automate.yml | 273 ++++++++++++++++++++++++++++++++++++ jjb/releng/docker-deploy.sh | 101 +++++++++++++ jjb/releng/docker-update.sh | 34 +++++ jjb/releng/testapi-automate.yml | 273 ------------------------------------ jjb/releng/testapi-docker-deploy.sh | 81 ----------- jjb/releng/testapi-docker-update.sh | 18 --- 6 files changed, 408 insertions(+), 372 deletions(-) create mode 100644 jjb/releng/automate.yml create mode 100644 jjb/releng/docker-deploy.sh create mode 100644 jjb/releng/docker-update.sh delete mode 100644 jjb/releng/testapi-automate.yml delete mode 100644 jjb/releng/testapi-docker-deploy.sh delete mode 100644 jjb/releng/testapi-docker-update.sh (limited to 'jjb') diff --git a/jjb/releng/automate.yml b/jjb/releng/automate.yml new file mode 100644 index 000000000..9eb281de3 --- /dev/null +++ b/jjb/releng/automate.yml @@ -0,0 +1,273 @@ +- project: + name: utils-automate + stream: + - master: + branch: '{stream}' + + module: + - 'testapi' + - 'reporting' + + phase: + - 'docker-deploy': + slave-label: 'testresults' + - 'generate-doc' + + jobs: + - '{module}-automate-{stream}' + - '{module}-automate-{phase}-{stream}' + - '{module}-verify-{stream}' + + project: 'releng' + +- job: + name: 'testapi-mongodb-backup' + + parameters: + - label: + name: SLAVE_LABEL + default: 'testresults' + description: 'Slave label on Jenkins' + - project-parameter: + project: 'releng' + branch: 'master' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/releng + description: 'Git URL to use on this Jenkins Slave' + + scm: + - git-scm + + triggers: + - timed: '@weekly' + + builders: + - mongodb-backup + +- job-template: + name: '{module}-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: 'utils/test/{module}/**' + + builders: + - shell: | + bash ./utils/test/{module}/run_test.sh + + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + health-auto-update: "false" + stability-auto-update: "false" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 10 + unhealthy: 20 + failing: 30 + - method: + healthy: 50 + unhealthy: 40 + failing: 30 + +- job-template: + name: '{module}-automate-{stream}' + + project-type: multijob + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + option: 'project' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - string: + name: DOCKER_TAG + default: 'latest' + description: 'Tag name for {module} docker image' + - string: + name: MODULE_NAME + default: '{module}' + description: "Name of the module" + - 'opnfv-build-defaults' + + scm: + - git-scm + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: 'utils/test/{module}/**' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - docker-update + - multijob: + name: docker-deploy + condition: SUCCESSFUL + projects: + - name: '{module}-automate-docker-deploy-{stream}' + current-parameters: false + predefined-parameters: | + GIT_BASE=$GIT_BASE + node-label-name: SLAVE_LABEL + node-label: testresults + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: generate-doc + condition: SUCCESSFUL + projects: + - name: '{module}-automate-generate-doc-{stream}' + current-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + + publishers: + - 'email-publisher' + +- job-template: + name: '{module}-automate-{phase}-{stream}' + + properties: + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 120 + fail: true + + scm: + - git-scm + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{module}-automate-{phase}-macro' + +################################ +# job builders +################################ +- builder: + name: 'docker-update' + builders: + - shell: + !include-raw: ./docker-update.sh + +- builder: + name: 'testapi-automate-generate-doc-macro' + builders: + - 'testapi-doc-build' + - 'upload-doc-artifact' + +- builder: + name: 'testapi-doc-build' + builders: + - shell: | + bash ./utils/test/testapi/htmlize/doc-build.sh + +- builder: + name: 'upload-doc-artifact' + builders: + - shell: | + bash ./utils/test/testapi/htmlize/push-doc-artifact.sh + +- builder: + name: 'reporting-automate-generate-doc-macro' + builders: + - shell: echo "To Be Done" + +- builder: + name: 'testapi-automate-docker-deploy-macro' + builders: + - shell: | + bash ./jjb/releng/docker-deploy.sh 'sudo docker run -dti -p 8082:8000 -e mongodb_url=mongodb://172.17.0.1:27017 -e swagger_url=http://testresults.opnfv.org/test opnfv/testapi' "http://testresults.opnfv.org/test/swagger/APIs" +- builder: + name: 'reporting-automate-docker-deploy-macro' + builders: + - shell: | + bash ./jjb/releng/docker-deploy.sh 'sudo docker run -itd -p 8084:8000 -e SERVER_URL=http://testresults.opnfv.org/reporting2:8084 opnfv/reporting' "http://testresults.opnfv.org/reporting2/reporting/index.html" + +- builder: + name: mongodb-backup + builders: + - shell: | + bash ./jjb/releng/testapi-backup-mongodb.sh + +################################ +# job publishers +################################ + +- publisher: + name: 'email-publisher' + publishers: + - email: + recipients: rohitsakala@gmail.com feng.xiaowei@zte.com.cn morgan.richomme@orange.com + notify-every-unstable-build: false + send-to-individuals: true diff --git a/jjb/releng/docker-deploy.sh b/jjb/releng/docker-deploy.sh new file mode 100644 index 000000000..b3b930fc4 --- /dev/null +++ b/jjb/releng/docker-deploy.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one * +# or more contributor license agreements. See the NOTICE file * +# distributed with this work for additional information * +# regarding copyright ownership. The ASF licenses this file * +# to you under the Apache License, Version 2.0 (the * +# "License"); you may not use this file except in compliance * +# with the License. You may obtain a copy of the License at * +# * +# http://www.apache.org/licenses/LICENSE-2.0 * +# * +# Unless required by applicable law or agreed to in writing, * +# software distributed under the License is distributed on an * +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * +# KIND, either express or implied. See the License for the * +# specific language governing permissions and limitations * +# under the License. * + +# Assigning Variables +command=$1 +url=$2 + +function check() { + + # Verify hosted + sleep 5 + cmd=`curl -s --head --request GET ${url} | grep '200 OK' > /dev/null` + rc=$? + echo $rc + + if [[ $rc == 0 ]] + then + return 0 + else + return 1 + fi + +} + +echo "Getting contianer Id of the currently running one" +contId=$(sudo docker ps | grep "opnfv/testapi:latest" | awk '{print $1}') + +echo "Pulling the latest image" +sudo docker pull opnfv/testapi:latest + +echo "Deleting old containers of opnfv/testapi:old" +sudo docker ps -a | grep "opnfv/testapi" | grep "old" | awk '{print $1}' | xargs -r sudo docker rm -f + +echo "Deleting old images of opnfv/testapi:latest" +sudo docker images | grep "opnfv/testapi" | grep "old" | awk '{print $3}' | xargs -r sudo docker rmi -f + + +if [[ -z "$contId" ]] +then + echo "No running testapi container" + + echo "Removing stopped testapi containers in the previous iterations" + sudo docker ps -f status=exited | grep "opnfv_testapi" | awk '{print $1}' | xargs -r sudo docker rm -f +else + echo $contId + + echo "Get the image id of the currently running conatiner" + currImgId=$(sudo docker ps | grep "$contId" | awk '{print $2}') + echo $currImgId + + if [[ -z "$currImgId" ]] + then + echo "No image id found for the container id" + exit 1 + fi + + echo "Changing current image tag to old" + sudo docker tag "$currImgId" opnfv/testapi:old + + echo "Removing stopped testapi containers in the previous iteration" + sudo docker ps -f status=exited | grep "opnfv_testapi" | awk '{print $1}' | xargs -r sudo docker rm -f + + echo "Renaming the running container name to opnfv_testapi as to identify it." + sudo docker rename $contId opnfv_testapi + + echo "Stop the currently running container" + sudo docker stop $contId +fi + +echo "Running a container with the new image" +$command:latest + +if check; then + echo "TestResults Hosted." +else + echo "TestResults Hosting Failed" + if [[ $(sudo docker images | grep "opnfv/testapi" | grep "old" | awk '{print $3}') ]]; then + echo "Running old Image" + $command:old + exit 1 + fi +fi + +# Echo Images and Containers +sudo docker images +sudo docker ps -a diff --git a/jjb/releng/docker-update.sh b/jjb/releng/docker-update.sh new file mode 100644 index 000000000..559ac835c --- /dev/null +++ b/jjb/releng/docker-update.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one * +# or more contributor license agreements. See the NOTICE file * +# distributed with this work for additional information * +# regarding copyright ownership. The ASF licenses this file * +# to you under the Apache License, Version 2.0 (the * +# "License"); you may not use this file except in compliance * +# with the License. You may obtain a copy of the License at * +# * +# http://www.apache.org/licenses/LICENSE-2.0 * +# * +# Unless required by applicable law or agreed to in writing, * +# software distributed under the License is distributed on an * +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * +# KIND, either express or implied. See the License for the * +# specific language governing permissions and limitations * +# under the License. * + +set -o errexit +set -o nounset + +cd $WORKSPACE/utils/test/$MODULE_NAME/docker/ + +# Remove previous containers +docker ps -a | grep "opnfv/$MODULE_NAME" | awk '{ print $1 }' | xargs -r docker rm -f + +# Remove previous images +docker images | grep "opnfv/$MODULE_NAME" | awk '{ print $3 }' | xargs -r docker rmi -f + +# Start build +docker build --no-cache -t opnfv/$MODULE_NAME:$DOCKER_TAG . + +# Push Image +docker push opnfv/$MODULE_NAME:$DOCKER_TAG diff --git a/jjb/releng/testapi-automate.yml b/jjb/releng/testapi-automate.yml deleted file mode 100644 index dd76538a3..000000000 --- a/jjb/releng/testapi-automate.yml +++ /dev/null @@ -1,273 +0,0 @@ -- project: - name: testapi-automate - stream: - - master: - branch: '{stream}' - gs-pathname: '' - - phase: - - 'docker-update' - - 'docker-deploy': - slave-label: 'testresults' - - 'generate-doc' - - jobs: - - 'testapi-automate-{stream}' - - 'testapi-automate-{phase}-{stream}' - - 'testapi-verify-{stream}' - - project: 'releng' - -- job: - name: 'testapi-mongodb-backup' - - parameters: - - label: - name: SLAVE_LABEL - default: 'testresults' - description: 'Slave label on Jenkins' - - project-parameter: - project: 'releng' - branch: 'master' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/releng - description: 'Git URL to use on this Jenkins Slave' - - scm: - - git-scm - - triggers: - - timed: '@weekly' - - builders: - - mongodb-backup - -- job-template: - name: 'testapi-verify-{stream}' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: 'ANT' - pattern: 'utils/test/testapi/**' - - builders: - - run-unit-tests - - publishers: - - junit: - results: nosetests.xml - - cobertura: - report-file: "coverage.xml" - only-stable: "true" - health-auto-update: "false" - stability-auto-update: "false" - zoom-coverage-chart: "true" - targets: - - files: - healthy: 10 - unhealthy: 20 - failing: 30 - - method: - healthy: 50 - unhealthy: 40 - failing: 30 - -- job-template: - name: 'testapi-automate-{stream}' - - project-type: multijob - - properties: - - throttle: - enabled: true - max-total: 1 - max-per-node: 1 - option: 'project' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - string: - name: DOCKER_TAG - default: "latest" - description: "Tag name for testapi docker image" - - 'opnfv-build-defaults' - - scm: - - git-scm - - wrappers: - - ssh-agent-wrapper - - timeout: - timeout: 360 - fail: true - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: 'ANT' - pattern: 'utils/test/testapi/**' - - builders: - - description-setter: - description: "Built on $NODE_NAME" - - multijob: - name: docker-update - condition: SUCCESSFUL - projects: - - name: 'testapi-automate-docker-update-{stream}' - current-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - - multijob: - name: docker-deploy - condition: SUCCESSFUL - projects: - - name: 'testapi-automate-docker-deploy-{stream}' - current-parameters: false - predefined-parameters: | - GIT_BASE=$GIT_BASE - node-label-name: SLAVE_LABEL - node-label: testresults - kill-phase-on: FAILURE - abort-all-job: true - - multijob: - name: generate-doc - condition: SUCCESSFUL - projects: - - name: 'testapi-automate-generate-doc-{stream}' - current-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - - publishers: - - 'email-publisher' - -- job-template: - name: 'testapi-automate-{phase}-{stream}' - - properties: - - throttle: - enabled: true - max-per-node: 1 - option: 'project' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - string: - name: DOCKER_TAG - default: "latest" - description: "Tag name for testapi docker image" - - wrappers: - - ssh-agent-wrapper - - timeout: - timeout: 120 - fail: true - - scm: - - git-scm - - builders: - - description-setter: - description: "Built on $NODE_NAME" - - 'testapi-automate-{phase}-macro' - -################################ -# job builders -################################ -- builder: - name: mongodb-backup - builders: - - shell: | - bash ./jjb/releng/testapi-backup-mongodb.sh - -- builder: - name: 'run-unit-tests' - builders: - - shell: | - bash ./utils/test/testapi/run_test.sh - -- builder: - name: 'testapi-automate-docker-update-macro' - builders: - - shell: | - bash ./jjb/releng/testapi-docker-update.sh - -- builder: - name: 'testapi-automate-generate-doc-macro' - builders: - - 'testapi-doc-build' - - 'upload-doc-artifact' - -- builder: - name: 'testapi-doc-build' - builders: - - shell: | - bash ./utils/test/testapi/htmlize/doc-build.sh - -- builder: - name: 'upload-doc-artifact' - builders: - - shell: | - bash ./utils/test/testapi/htmlize/push-doc-artifact.sh - -- builder: - name: 'testapi-automate-docker-deploy-macro' - builders: - - shell: | - bash ./jjb/releng/testapi-docker-deploy.sh - -################################ -# job publishers -################################ - -- publisher: - name: 'email-publisher' - publishers: - - email: - recipients: rohitsakala@gmail.com feng.xiaowei@zte.com.cn - notify-every-unstable-build: false - send-to-individuals: true diff --git a/jjb/releng/testapi-docker-deploy.sh b/jjb/releng/testapi-docker-deploy.sh deleted file mode 100644 index 02c5e3a27..000000000 --- a/jjb/releng/testapi-docker-deploy.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -function check() { - - # Verify hosted - sleep 5 - cmd=`curl -s --head --request GET http://testresults.opnfv.org/test/swagger/APIs | grep '200 OK' > /dev/null` - rc=$? - echo $rc - - if [[ $rc == 0 ]] - then - return 0 - else - return 1 - fi - -} - -echo "Getting contianer Id of the currently running one" -contId=$(sudo docker ps | grep "opnfv/testapi:latest" | awk '{print $1}') - -echo "Pulling the latest image" -sudo docker pull opnfv/testapi:latest - -echo "Deleting old containers of opnfv/testapi:old" -sudo docker ps -a | grep "opnfv/testapi" | grep "old" | awk '{print $1}' | xargs -r sudo docker rm -f - -echo "Deleting old images of opnfv/testapi:latest" -sudo docker images | grep "opnfv/testapi" | grep "old" | awk '{print $3}' | xargs -r sudo docker rmi -f - - -if [[ -z "$contId" ]] -then - echo "No running testapi container" - - echo "Removing stopped testapi containers in the previous iterations" - sudo docker ps -f status=exited | grep "opnfv_testapi" | awk '{print $1}' | xargs -r sudo docker rm -f -else - echo $contId - - echo "Get the image id of the currently running conatiner" - currImgId=$(sudo docker ps | grep "$contId" | awk '{print $2}') - echo $currImgId - - if [[ -z "$currImgId" ]] - then - echo "No image id found for the container id" - exit 1 - fi - - echo "Changing current image tag to old" - sudo docker tag "$currImgId" opnfv/testapi:old - - echo "Removing stopped testapi containers in the previous iteration" - sudo docker ps -f status=exited | grep "opnfv_testapi" | awk '{print $1}' | xargs -r sudo docker rm -f - - echo "Renaming the running container name to opnfv_testapi as to identify it." - sudo docker rename $contId opnfv_testapi - - echo "Stop the currently running container" - sudo docker stop $contId -fi - -echo "Running a container with the new image" -sudo docker run -dti -p "8082:8000" -e "mongodb_url=mongodb://172.17.0.1:27017" -e "swagger_url=http://testresults.opnfv.org/test" opnfv/testapi:latest - -if check; then - echo "TestResults Hosted." -else - echo "TestResults Hosting Failed" - if [[ $(sudo docker images | grep "opnfv/testapi" | grep "old" | awk '{print $3}') ]]; then - echo "Running old Image" - sudo docker run -dti -p "8082:8000" -e "mongodb_url=mongodb://172.17.0.1:27017" -e "swagger_url=http://testresults.opnfv.org/test" opnfv/testapi:old - exit 1 - fi -fi - -# Echo Images and Containers -sudo docker images -sudo docker ps -a diff --git a/jjb/releng/testapi-docker-update.sh b/jjb/releng/testapi-docker-update.sh deleted file mode 100644 index 84f5c3217..000000000 --- a/jjb/releng/testapi-docker-update.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset - -cd $WORKSPACE/utils/test/testapi/docker/ - -# Remove previous containers -docker ps -a | grep "opnfv/testapi" | awk '{ print $1 }' | xargs -r docker rm -f - -# Remove previous images -docker images | grep "opnfv/testapi" | awk '{ print $3 }' | xargs -r docker rmi -f - -# Start build -docker build --no-cache -t opnfv/testapi:$DOCKER_TAG . - -# Push Image -docker push opnfv/testapi:$DOCKER_TAG -- cgit 1.2.3-korg