From d8f85fabb2be5fea3630821bc8a9f186781ce2b9 Mon Sep 17 00:00:00 2001 From: Guo Ruijing Date: Thu, 9 Nov 2017 12:35:02 -0500 Subject: fix container4nfv jenkins job hung issue 1. fix cleanup: poweroff/delete all vms 2. add timeout for kubeadm init/join hung: https://github.com/kubernetes/kubernetes/issues/43800 3. try to fix kubeadm init/join hung Change-Id: I133d5aafee0455848909484efee5548b2015473d Signed-off-by: Guo Ruijing --- ci/cleanup.sh | 2 +- src/vagrant/kubeadm_basic/host_setup.sh | 5 +++++ src/vagrant/kubeadm_basic/master_setup.sh | 4 +++- src/vagrant/kubeadm_basic/worker_setup.sh | 2 +- src/vagrant/kubeadm_multus/host_setup.sh | 4 ++++ src/vagrant/kubeadm_multus/master_setup.sh | 4 +++- src/vagrant/kubeadm_multus/worker_setup.sh | 2 +- src/vagrant/kubeadm_ovsdpdk/host_setup.sh | 5 +++++ src/vagrant/kubeadm_ovsdpdk/master_setup.sh | 4 +++- src/vagrant/kubeadm_ovsdpdk/worker_setup.sh | 2 +- src/vagrant/kubeadm_virtlet/host_setup.sh | 4 ++++ src/vagrant/kubeadm_virtlet/master_setup.sh | 4 +++- src/vagrant/kubeadm_virtlet/worker_setup.sh | 2 +- 13 files changed, 35 insertions(+), 9 deletions(-) diff --git a/ci/cleanup.sh b/ci/cleanup.sh index ee5c428..afe4ac0 100755 --- a/ci/cleanup.sh +++ b/ci/cleanup.sh @@ -17,7 +17,7 @@ set -x -VBoxManage list vms | cut -f2 -d'"' | xargs -I {} VBoxManage unregistervm {} --delete VBoxManage list vms | cut -f2 -d'"' | xargs -I {} VBoxManage controlvm {} poweroff +VBoxManage list vms | cut -f2 -d'"' | xargs -I {} VBoxManage unregistervm {} --delete VBoxManage list hostonlyifs | grep "^Name:.*vboxnet" |\ sed "s/^Name:.*vboxnet/vboxnet/" | xargs -I {} VBoxManage hostonlyif remove {} diff --git a/src/vagrant/kubeadm_basic/host_setup.sh b/src/vagrant/kubeadm_basic/host_setup.sh index 990df7f..1aad518 100644 --- a/src/vagrant/kubeadm_basic/host_setup.sh +++ b/src/vagrant/kubeadm_basic/host_setup.sh @@ -17,3 +17,8 @@ EOF sudo apt-get update sudo apt-get install -y docker.io sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00 + +sudo rm -rf /var/lib/kubelet +sudo systemctl stop kubelet +sudo systemctl daemon-reload +sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm_basic/master_setup.sh b/src/vagrant/kubeadm_basic/master_setup.sh index 34c5923..bcbda09 100644 --- a/src/vagrant/kubeadm_basic/master_setup.sh +++ b/src/vagrant/kubeadm_basic/master_setup.sh @@ -1,6 +1,8 @@ #!/bin/bash -sudo kubeadm init --apiserver-advertise-address 192.168.1.10 --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 +set -ex + +sudo timeout 3600 kubeadm init --apiserver-advertise-address 192.168.1.10 --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 sudo cp /etc/kubernetes/admin.conf $HOME/ sudo chown $(id -u):$(id -g) $HOME/admin.conf export KUBECONFIG=$HOME/admin.conf diff --git a/src/vagrant/kubeadm_basic/worker_setup.sh b/src/vagrant/kubeadm_basic/worker_setup.sh index b68d800..8948ad5 100644 --- a/src/vagrant/kubeadm_basic/worker_setup.sh +++ b/src/vagrant/kubeadm_basic/worker_setup.sh @@ -1,4 +1,4 @@ #!/bin/bash set -ex -sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true +sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true diff --git a/src/vagrant/kubeadm_multus/host_setup.sh b/src/vagrant/kubeadm_multus/host_setup.sh index 003bf2b..d9b68e2 100644 --- a/src/vagrant/kubeadm_multus/host_setup.sh +++ b/src/vagrant/kubeadm_multus/host_setup.sh @@ -17,4 +17,8 @@ EOF sudo apt-get update sudo apt-get install -y docker.io sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00 + sudo rm -rf /var/lib/kubelet +sudo systemctl stop kubelet +sudo systemctl daemon-reload +sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm_multus/master_setup.sh b/src/vagrant/kubeadm_multus/master_setup.sh index 7d3dc3e..8170f92 100644 --- a/src/vagrant/kubeadm_multus/master_setup.sh +++ b/src/vagrant/kubeadm_multus/master_setup.sh @@ -1,6 +1,8 @@ #!/bin/bash -sudo kubeadm init --apiserver-advertise-address=10.96.0.10 --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 +set -ex + +sudo timeout 3600 kubeadm init --apiserver-advertise-address=10.96.0.10 --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 sudo cp /etc/kubernetes/admin.conf $HOME/ sudo chown $(id -u):$(id -g) $HOME/admin.conf export KUBECONFIG=$HOME/admin.conf diff --git a/src/vagrant/kubeadm_multus/worker_setup.sh b/src/vagrant/kubeadm_multus/worker_setup.sh index 1ba3ec8..19e1051 100644 --- a/src/vagrant/kubeadm_multus/worker_setup.sh +++ b/src/vagrant/kubeadm_multus/worker_setup.sh @@ -1,4 +1,4 @@ #!/bin/bash set -ex -sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true +sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true diff --git a/src/vagrant/kubeadm_ovsdpdk/host_setup.sh b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh index 990df7f..1aad518 100644 --- a/src/vagrant/kubeadm_ovsdpdk/host_setup.sh +++ b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh @@ -17,3 +17,8 @@ EOF sudo apt-get update sudo apt-get install -y docker.io sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00 + +sudo rm -rf /var/lib/kubelet +sudo systemctl stop kubelet +sudo systemctl daemon-reload +sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm_ovsdpdk/master_setup.sh b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh index f5bb6fc..9bd06f6 100644 --- a/src/vagrant/kubeadm_ovsdpdk/master_setup.sh +++ b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh @@ -1,6 +1,8 @@ #!/bin/bash -sudo kubeadm init --apiserver-advertise-address 192.168.1.10 --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 +set -ex + +sudo timeout 3600 kubeadm init --apiserver-advertise-address 192.168.1.10 --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 sudo cp /etc/kubernetes/admin.conf $HOME/ sudo chown $(id -u):$(id -g) $HOME/admin.conf export KUBECONFIG=$HOME/admin.conf diff --git a/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh index b68d800..8948ad5 100644 --- a/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh +++ b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh @@ -1,4 +1,4 @@ #!/bin/bash set -ex -sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true +sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true diff --git a/src/vagrant/kubeadm_virtlet/host_setup.sh b/src/vagrant/kubeadm_virtlet/host_setup.sh index 003bf2b..d9b68e2 100644 --- a/src/vagrant/kubeadm_virtlet/host_setup.sh +++ b/src/vagrant/kubeadm_virtlet/host_setup.sh @@ -17,4 +17,8 @@ EOF sudo apt-get update sudo apt-get install -y docker.io sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00 + sudo rm -rf /var/lib/kubelet +sudo systemctl stop kubelet +sudo systemctl daemon-reload +sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm_virtlet/master_setup.sh b/src/vagrant/kubeadm_virtlet/master_setup.sh index 5ae2487..8831782 100644 --- a/src/vagrant/kubeadm_virtlet/master_setup.sh +++ b/src/vagrant/kubeadm_virtlet/master_setup.sh @@ -1,6 +1,8 @@ #!/bin/bash -sudo kubeadm init --apiserver-advertise-address=10.96.0.10 --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 +set -ex + +sudo timeout 3600 kubeadm init --apiserver-advertise-address=10.96.0.10 --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 sudo cp /etc/kubernetes/admin.conf $HOME/ sudo chown $(id -u):$(id -g) $HOME/admin.conf export KUBECONFIG=$HOME/admin.conf diff --git a/src/vagrant/kubeadm_virtlet/worker_setup.sh b/src/vagrant/kubeadm_virtlet/worker_setup.sh index bc28dab..b250350 100644 --- a/src/vagrant/kubeadm_virtlet/worker_setup.sh +++ b/src/vagrant/kubeadm_virtlet/worker_setup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex -sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true +sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true sudo docker run --rm openretriever/virtlet tar -c /criproxy | sudo tar -C /usr/local/bin -xv sudo ln -s /usr/local/bin/criproxy /usr/local/bin/dockershim -- cgit 1.2.3-korg