diff options
Diffstat (limited to 'utils/test/opts/watchdog.sh')
| -rw-r--r-- | utils/test/opts/watchdog.sh | 162 |
1 files changed, 0 insertions, 162 deletions
diff --git a/utils/test/opts/watchdog.sh b/utils/test/opts/watchdog.sh deleted file mode 100644 index 51868d709..000000000 --- a/utils/test/opts/watchdog.sh +++ /dev/null @@ -1,162 +0,0 @@ -# * -# http://www.apache.org/licenses/LICENSE-2.0 * -# * -# Unless required by applicable law or agreed to in writing, * -# software distributed under the License is distributed on an * -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * -# KIND, either express or implied. See the License for the * -# specific language governing permissions and limitations * -# under the License. * - -# This script checks if deployments are working or and then -# starts the specified containers in case one of the containers -# crash. The only solution is restarting docker as of now. - -#!/bin/bash - -## List of modules -modules=(testapi reporting) - -## Ports of the modules -declare -A ports=( ["testapi"]="8082" ["reporting"]="8084") - -## Urls to check if the modules are deployed or not ? -declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \ - ["reporting"]="http://testresults.opnfv.org/reporting/index.html") - -### Functions related to checking. - -function is_deploying() { - xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-docker-deploy-master/lastBuild/api/xml?depth=1") - building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml") - if [[ $building == "false" ]] - then - false - else - true - fi -} - -function get_docker_status() { - status=$(service docker status | sed -n 3p | cut -d ' ' -f5) - echo -e "Docker status: $status" - if [ $status = "active" ] - then - true - else - false - fi -} - -function check_connectivity() { - echo "Checking $1 connection : $2" - cmd=`curl --head -m10 --request GET ${2} | grep '200 OK' > /dev/null` - rc=$? - if [[ $rc == 0 ]]; then - true - else - false - fi -} - -function check_modules() { - echo -e "Checking modules" - failed_modules=() - for module in "${modules[@]}" - do - if ! check_connectivity $module "${urls[$module]}"; then - if ! is_deploying $module; then - echo -e "$module failed" - failed_modules+=($module) - fi - fi - done - if [ ! -z "$failed_modules" ]; then - echo -e "Failed Modules: $failed_modules" - false - else - echo -e "All modules working good" - exit 0 - fi -} - -### Functions related fixes. - -function restart_docker_fix() { - echo -e "Running restart_docker_fix" - service docker restart - start_containers_fix "${modules[@]}" -} - -function docker_proxy_fix() { - echo -e "Running docker_proxy_fix" - fix_modules=("${@}") - for module in "${fix_modules[@]}" - do - echo -e "Kill docker proxy and restart containers" - pid=$(netstat -nlp | grep :${ports[$module]} | awk '{print $7}' | cut -d'/' -f1) - echo $pid - if [ ! -z "$pid" ]; then - kill $pid - start_container_fix $module - fi - done -} - -function start_containers_fix() { - start_modules=("${@}") - for module in "${start_modules[@]}" - do - start_container_fix $module - done -} - -function start_container_fix() { - echo -e "Starting a container $module" - sudo docker restart $module - sleep 5 - if ! check_connectivity $module "${urls[$module]}"; then - echo -e "Starting an old container $module_old" - sudo docker restart $module"_old" - sleep 5 - fi -} - -### Main Flow - -echo -e -echo -e "WatchDog Started" -echo -e -echo -e `date "+%Y-%m-%d %H:%M:%S.%N"` -echo -e - -## If the problem is related to docker daemon - -if ! get_docker_status; then - restart_docker_fix - if ! check_modules; then - echo -e "Watchdog failed while restart_docker_fix" - fi - exit -fi - -## If the problem is related to docker proxy - -if ! check_modules; then - docker_proxy_fix "${failed_modules[@]}" -fi - -## If any other problem : restart docker - -if ! check_modules; then - restart_docker_fix -fi - -## If nothing works out - -if ! check_modules; then - echo -e "Watchdog failed" -fi - -sudo docker ps -sudo docker images
\ No newline at end of file |