fix container4nfv jenkins job hung issue

1. fix cleanup: poweroff/delete all vms
2. add timeout for kubeadm init/join hung:
    https://github.com/kubernetes/kubernetes/issues/43800
3. try to fix kubeadm init/join hung

Change-Id: I133d5aafee0455848909484efee5548b2015473d
Signed-off-by: Guo Ruijing <ruijing.guo@intel.com>

13 files changed, 35 insertions, 9 deletions

diff --git a/ci/cleanup.sh b/ci/cleanup.sh
index ee5c428..afe4ac0 100755
--- a/ci/cleanup.sh
+++ b/ci/cleanup.sh
@@ -17,7 +17,7 @@
 
 set -x
 
-VBoxManage list vms | cut -f2 -d'"' | xargs -I {} VBoxManage unregistervm {} --delete
 VBoxManage list vms | cut -f2 -d'"' | xargs -I {} VBoxManage controlvm {} poweroff
+VBoxManage list vms | cut -f2 -d'"' | xargs -I {} VBoxManage unregistervm {} --delete
 VBoxManage list hostonlyifs | grep "^Name:.*vboxnet" |\
     sed "s/^Name:.*vboxnet/vboxnet/" | xargs -I {} VBoxManage hostonlyif remove {}
diff --git a/src/vagrant/kubeadm_basic/host_setup.sh b/src/vagrant/kubeadm_basic/host_setup.sh
index 990df7f..1aad518 100644
--- a/src/vagrant/kubeadm_basic/host_setup.sh
+++ b/src/vagrant/kubeadm_basic/host_setup.sh
@@ -17,3 +17,8 @@ EOF
 sudo apt-get update
 sudo apt-get install -y docker.io
 sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
+
+sudo rm -rf /var/lib/kubelet
+sudo systemctl stop kubelet
+sudo systemctl daemon-reload
+sudo systemctl start kubelet
diff --git a/src/vagrant/kubeadm_basic/master_setup.sh b/src/vagrant/kubeadm_basic/master_setup.sh
index 34c5923..bcbda09 100644
--- a/src/vagrant/kubeadm_basic/master_setup.sh
+++ b/src/vagrant/kubeadm_basic/master_setup.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
-sudo kubeadm init --apiserver-advertise-address 192.168.1.10  --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
+set -ex
+
+sudo timeout 3600 kubeadm init --apiserver-advertise-address 192.168.1.10  --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
 sudo cp /etc/kubernetes/admin.conf $HOME/
 sudo chown $(id -u):$(id -g) $HOME/admin.conf
 export KUBECONFIG=$HOME/admin.conf
diff --git a/src/vagrant/kubeadm_basic/worker_setup.sh b/src/vagrant/kubeadm_basic/worker_setup.sh
index b68d800..8948ad5 100644
--- a/src/vagrant/kubeadm_basic/worker_setup.sh
+++ b/src/vagrant/kubeadm_basic/worker_setup.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 set -ex
-sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true
+sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true
diff --git a/src/vagrant/kubeadm_multus/host_setup.sh b/src/vagrant/kubeadm_multus/host_setup.sh
index 003bf2b..d9b68e2 100644
--- a/src/vagrant/kubeadm_multus/host_setup.sh
+++ b/src/vagrant/kubeadm_multus/host_setup.sh
@@ -17,4 +17,8 @@ EOF
 sudo apt-get update
 sudo apt-get install -y docker.io
 sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
+
 sudo rm -rf /var/lib/kubelet
+sudo systemctl stop kubelet
+sudo systemctl daemon-reload
+sudo systemctl start kubelet
diff --git a/src/vagrant/kubeadm_multus/master_setup.sh b/src/vagrant/kubeadm_multus/master_setup.sh
index 7d3dc3e..8170f92 100644
--- a/src/vagrant/kubeadm_multus/master_setup.sh
+++ b/src/vagrant/kubeadm_multus/master_setup.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
-sudo kubeadm init --apiserver-advertise-address=10.96.0.10  --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
+set -ex
+
+sudo timeout 3600 kubeadm init --apiserver-advertise-address=10.96.0.10  --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
 sudo cp /etc/kubernetes/admin.conf $HOME/
 sudo chown $(id -u):$(id -g) $HOME/admin.conf
 export KUBECONFIG=$HOME/admin.conf
diff --git a/src/vagrant/kubeadm_multus/worker_setup.sh b/src/vagrant/kubeadm_multus/worker_setup.sh
index 1ba3ec8..19e1051 100644
--- a/src/vagrant/kubeadm_multus/worker_setup.sh
+++ b/src/vagrant/kubeadm_multus/worker_setup.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 set -ex
-sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true
+sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true
diff --git a/src/vagrant/kubeadm_ovsdpdk/host_setup.sh b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh
index 990df7f..1aad518 100644
--- a/src/vagrant/kubeadm_ovsdpdk/host_setup.sh
+++ b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh
@@ -17,3 +17,8 @@ EOF
 sudo apt-get update
 sudo apt-get install -y docker.io
 sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
+
+sudo rm -rf /var/lib/kubelet
+sudo systemctl stop kubelet
+sudo systemctl daemon-reload
+sudo systemctl start kubelet
diff --git a/src/vagrant/kubeadm_ovsdpdk/master_setup.sh b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh
index f5bb6fc..9bd06f6 100644
--- a/src/vagrant/kubeadm_ovsdpdk/master_setup.sh
+++ b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
-sudo kubeadm init --apiserver-advertise-address 192.168.1.10  --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
+set -ex
+
+sudo timeout 3600 kubeadm init --apiserver-advertise-address 192.168.1.10  --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
 sudo cp /etc/kubernetes/admin.conf $HOME/
 sudo chown $(id -u):$(id -g) $HOME/admin.conf
 export KUBECONFIG=$HOME/admin.conf
diff --git a/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh
index b68d800..8948ad5 100644
--- a/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh
+++ b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 set -ex
-sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true
+sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true
diff --git a/src/vagrant/kubeadm_virtlet/host_setup.sh b/src/vagrant/kubeadm_virtlet/host_setup.sh
index 003bf2b..d9b68e2 100644
--- a/src/vagrant/kubeadm_virtlet/host_setup.sh
+++ b/src/vagrant/kubeadm_virtlet/host_setup.sh
@@ -17,4 +17,8 @@ EOF
 sudo apt-get update
 sudo apt-get install -y docker.io
 sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
+
 sudo rm -rf /var/lib/kubelet
+sudo systemctl stop kubelet
+sudo systemctl daemon-reload
+sudo systemctl start kubelet
diff --git a/src/vagrant/kubeadm_virtlet/master_setup.sh b/src/vagrant/kubeadm_virtlet/master_setup.sh
index 5ae2487..8831782 100644
--- a/src/vagrant/kubeadm_virtlet/master_setup.sh
+++ b/src/vagrant/kubeadm_virtlet/master_setup.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
-sudo kubeadm init --apiserver-advertise-address=10.96.0.10  --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
+set -ex
+
+sudo timeout 3600 kubeadm init --apiserver-advertise-address=10.96.0.10  --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
 sudo cp /etc/kubernetes/admin.conf $HOME/
 sudo chown $(id -u):$(id -g) $HOME/admin.conf
 export KUBECONFIG=$HOME/admin.conf
diff --git a/src/vagrant/kubeadm_virtlet/worker_setup.sh b/src/vagrant/kubeadm_virtlet/worker_setup.sh
index bc28dab..b250350 100644
--- a/src/vagrant/kubeadm_virtlet/worker_setup.sh
+++ b/src/vagrant/kubeadm_virtlet/worker_setup.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 set -ex
-sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true
+sudo timeout 3600 kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true
 
 sudo docker run --rm openretriever/virtlet tar -c /criproxy | sudo tar -C /usr/local/bin -xv
 sudo ln -s /usr/local/bin/criproxy /usr/local/bin/dockershim