diff options
| -rwxr-xr-x | jjb/apex/apex-download-artifact.sh | 2 | ||||
| -rw-r--r-- | jjb/auto/auto.yml | 56 | ||||
| -rw-r--r-- | jjb/container4nfv/container4nfv-project.yml (renamed from jjb/openretriever/openretriever-project.yml) | 6 | ||||
| -rw-r--r-- | jjb/dpacc/dpacc.yml | 77 | ||||
| -rw-r--r-- | jjb/opnfvdocs/project.cfg | 2 | ||||
| -rw-r--r-- | jjb/xci/osa-periodic-jobs.yml | 6 | ||||
| -rw-r--r-- | jjb/xci/xci-daily-jobs.yml | 12 | ||||
| -rwxr-xr-x | jjb/yardstick/yardstick-daily.sh | 2 | ||||
| -rwxr-xr-x | utils/fetch_os_creds.sh | 7 | ||||
| -rw-r--r-- | utils/test/testapi/tools/watchdog/docker_watch.sh | 178 |
10 files changed, 296 insertions, 52 deletions
diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh index 7293e8243..a11fb65b8 100755 --- a/jjb/apex/apex-download-artifact.sh +++ b/jjb/apex/apex-download-artifact.sh @@ -52,7 +52,7 @@ else mkdir -p ~/apex_rpms pushd ~/apex_rpms # Remove older rpms which do not match this version - find . ! -name *${VERSION_EXTENSION}.noarch.rpm -type f -exec rm -f {} + + find . ! -name "*${VERSION_EXTENSION}.noarch.rpm" -type f -exec rm -f {} + # Download RPM only if changed on server for rpm in $RPM_LIST; do wget -N ${RPM_INSTALL_PATH}/${rpm} diff --git a/jjb/auto/auto.yml b/jjb/auto/auto.yml new file mode 100644 index 000000000..fefa37626 --- /dev/null +++ b/jjb/auto/auto.yml @@ -0,0 +1,56 @@ +--- +- project: + name: auto + + project: '{name}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + + jobs: + - 'auto-verify-{stream}' + +- job-template: + name: 'auto-verify-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - shell: | + echo "Nothing to verify!" diff --git a/jjb/openretriever/openretriever-project.yml b/jjb/container4nfv/container4nfv-project.yml index 3bcfab6d3..3b29b36f8 100644 --- a/jjb/openretriever/openretriever-project.yml +++ b/jjb/container4nfv/container4nfv-project.yml @@ -3,12 +3,12 @@ # They will only be enabled on request by projects! ################################################### - project: - name: openretriever + name: container4nfv project: '{name}' jobs: - - 'openretriever-verify-{stream}' + - 'container4nfv-verify-{stream}' stream: - master: @@ -21,7 +21,7 @@ disabled: false - job-template: - name: 'openretriever-verify-{stream}' + name: 'container4nfv-verify-{stream}' disabled: '{obj:disabled}' diff --git a/jjb/dpacc/dpacc.yml b/jjb/dpacc/dpacc.yml index 63eb044ad..39f3d563c 100644 --- a/jjb/dpacc/dpacc.yml +++ b/jjb/dpacc/dpacc.yml @@ -1,3 +1,4 @@ +--- ################################################### # All the jobs except verify have been removed! # They will only be enabled on request by projects! @@ -8,17 +9,17 @@ project: '{name}' jobs: - - 'dpacc-verify-{stream}' + - 'dpacc-verify-{stream}' stream: - - master: - branch: '{stream}' - gs-pathname: '' - disabled: false - - danube: - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - disabled: false + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - danube: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false - job-template: name: 'dpacc-verify-{stream}' @@ -26,38 +27,38 @@ disabled: '{obj:disabled}' parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' scm: - - git-scm-gerrit + - git-scm-gerrit triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'true' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**|.gitignore' + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' builders: - - shell: | - echo "Nothing to verify!" + - shell: | + echo "Nothing to verify!" diff --git a/jjb/opnfvdocs/project.cfg b/jjb/opnfvdocs/project.cfg index 1ea05c1d4..0722b4036 100644 --- a/jjb/opnfvdocs/project.cfg +++ b/jjb/opnfvdocs/project.cfg @@ -5,6 +5,7 @@ bottlenecks compass4nfv copper conductor +container4nfv daisy doctor domino @@ -24,7 +25,6 @@ movie multisite octopus onosfw -openretriever ovno ovsnfv parser diff --git a/jjb/xci/osa-periodic-jobs.yml b/jjb/xci/osa-periodic-jobs.yml index 833575039..7311baab4 100644 --- a/jjb/xci/osa-periodic-jobs.yml +++ b/jjb/xci/osa-periodic-jobs.yml @@ -127,8 +127,10 @@ - 'xci-verify-healthcheck-.*' - 'bifrost-verify-.*' - 'bifrost-periodic-.*' - - 'osa-verify-.*' - - 'osa-periodic-.*' + - 'osa-verify-deploy-.*' + - 'osa-verify-halthcheck-.*' + - 'osa-periodic-deploy-.*' + - 'osa-periodic-healthcheck-.*' block-level: 'NODE' parameters: diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml index 0257b28e1..c39d07e39 100644 --- a/jjb/xci/xci-daily-jobs.yml +++ b/jjb/xci/xci-daily-jobs.yml @@ -16,10 +16,6 @@ stream: master opnfv-releng-version: master gs-pathname: '' - ocata: &ocata - stream: ocata - opnfv-releng-version: master - gs-pathname: '/{stream}' #-------------------------------- # Scenarios #-------------------------------- @@ -30,14 +26,18 @@ - 'os-nosdn-nofeature-noha': auto-trigger-name: 'daily-trigger-disabled' xci-flavor: 'noha' + - 'os-odl-sfc-ha': + auto-trigger-name: 'daily-trigger-disabled' + xci-flavor: 'ha' + - 'os-odl-sfc-noha': + auto-trigger-name: 'daily-trigger-disabled' + xci-flavor: 'noha' #-------------------------------- # XCI PODs #-------------------------------- pod: - virtual: <<: *master - - virtual: - <<: *ocata #-------------------------------- # Supported Distros #-------------------------------- diff --git a/jjb/yardstick/yardstick-daily.sh b/jjb/yardstick/yardstick-daily.sh index c91ea47a1..aaefba26e 100755 --- a/jjb/yardstick/yardstick-daily.sh +++ b/jjb/yardstick/yardstick-daily.sh @@ -60,7 +60,7 @@ map_log_dir="-v ${dir_result}:/tmp/yardstick" # Run docker if [[ ${INSTALLER_TYPE} == "joid" && "${DEPLOY_SCENARIO:0:2}" == "k8" ]];then - juju ssh kubernetes-master/0 sudo apt-get install docker.io + juju ssh kubernetes-master/0 sudo apt-get install -y docker.io cmd="juju ssh kubernetes-master/0 sudo docker run ${opts} ${envs} ${rc_file_vol} ${cacert_file_vol} ${map_log_dir} ${sshkey} opnfv/yardstick:${DOCKER_TAG} exec_tests.sh ${YARDSTICK_DB_BACKEND} ${YARDSTICK_SCENARIO_SUITE_NAME}" else cmd="sudo docker run ${opts} ${envs} ${rc_file_vol} ${cacert_file_vol} ${map_log_dir} ${sshkey} opnfv/yardstick:${DOCKER_TAG} \ diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh index 377930d66..0e2a2b93f 100755 --- a/utils/fetch_os_creds.sh +++ b/utils/fetch_os_creds.sh @@ -148,6 +148,13 @@ if [ "$installer_type" == "fuel" ]; then echo $auth_url >> $dest_path elif [ "$installer_type" == "apex" ]; then + if ! ipcalc -c $installer_ip; then + installer_ip=$(sudo virsh domifaddr undercloud | grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') + if [ -z "$installer_ip" ] || ! $(ipcalc -c $installer_ip); then + echo "Unable to find valid IP for Apex undercloud: ${installer_ip}" + exit 1 + fi + fi verify_connectivity $installer_ip # The credentials file is located in the Instack VM (192.0.2.1) diff --git a/utils/test/testapi/tools/watchdog/docker_watch.sh b/utils/test/testapi/tools/watchdog/docker_watch.sh new file mode 100644 index 000000000..d67e4b380 --- /dev/null +++ b/utils/test/testapi/tools/watchdog/docker_watch.sh @@ -0,0 +1,178 @@ +# * +# http://www.apache.org/licenses/LICENSE-2.0 * +# * +# Unless required by applicable law or agreed to in writing, * +# software distributed under the License is distributed on an * +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * +# KIND, either express or implied. See the License for the * +# specific language governing permissions and limitations * +# under the License. * + +# This script checks if deployments are working or and then +# starts the specified containers in case one of the containers +# crash. The only solution is restarting docker as of now. + +#!/bin/bash + +## List of modules +modules=(testapi reporting) + +## Ports of the modules +declare -A ports=( ["testapi"]="8082" ["reporting"]="8084") + +## Urls to check if the modules are deployed or not ? +#declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \ +# ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html") + +declare -A urls=( ["testapi"]="http://localhost:8082/" \ + ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html") + + +### Functions related to checking. + +function is_deploying() { + echo -e "Checking job statuses" + for module in "${modules[@]}" + do + if get_status $module; then + exit 0 + fi + done +} + +function get_status() { + xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-master/lastBuild/api/xml?depth=1") + building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml") + if [[ $building == "false" ]] + then + return 1 + else + return 0 + fi +} + +function get_docker_status() { + status=$(service docker status | sed -n 3p | cut -d ' ' -f5) + echo -e "Docker status: $status" + if [ $status = "active" ] + then + return 1 + else + return 0 + fi +} + +function check_connectivity() { + echo "Checking $1 connection : $2" + cmd=`curl --head -m10 --request GET ${2} | grep '200 OK' > /dev/null` + rc=$? + if [[ $rc == 0 ]]; then + return 0 + else + return 1 + fi +} + +function check_modules() { + echo -e "Checking modules" + failed_modules=() + for module in "${modules[@]}" + do + if ! check_connectivity $module "${urls[$module]}"; then + echo -e "$module failed" + failed_modules+=($module) + fi + done + if [ ! -z "$failed_modules" ]; then + echo -e "Failed Modules: $failed_modules" + return 1 + else + echo -e "All modules working good" + exit 0 + fi +} + +### Functions related fixes. + +function restart_docker_fix() { + echo -e "Running restart_docker_fix" + service docker restart + start_containers_fix "${modules[@]}" +} + +function docker_proxy_fix() { + echo -e "Running docker_proxy_fix" + fix_modules=("${@}") + for module in "${fix_modules[@]}" + do + echo -e "Kill docker proxy and restart containers" + pid=$(netstat -nlp | grep :${ports[$module]} | awk '{print $7}' | cut -d'/' -f1) + echo $pid + if [ ! -z "$pid" ]; then + kill $pid + start_containers_fix $module + fi + done +} + +function start_containers_fix() { + echo "Runnning start_containers_fix" + start_modules=("${@}") + for module in "${start_modules[@]}" + do + echo -e "Starting a container $module" + sudo docker stop $module + sudo docker start $module + sleep 5 + if ! check_connectivity $module "${urls[$module]}"; then + echo -e "Starting an old container $module_old" + sudo docker stop $module + sudo docker start $module"_old" + sleep 5 + fi + done +} + +### Main Flow + +echo -e +echo -e "WatchDog Started" +echo -e +echo -e `date "+%Y-%m-%d %H:%M:%S.%N"` +echo -e + +if ! is_deploying; then + echo -e "Jenkins Jobs running" + exit +fi + +## If the problem is related to docker daemon + +if get_docker_status; then + restart_docker_fix + if ! check_modules; then + echo -e "Watchdog failed while restart_docker_fix" + fi + exit +fi + +## If the problem is related to docker containers + +if ! check_modules; then + start_containers_fix "${failed_modules[@]}" +fi + +## If the problem is related to docker proxy + +if ! check_modules; then + docker_proxy_fix "${failed_modules[@]}" +fi + +## If nothing works out + +if ! check_modules; then + echo -e "Watchdog failed" +fi + +sudo docker ps +sudo docker images
\ No newline at end of file |