Update k8s and docker-ce versions

Add option to deploy k8s core only

JIRA: MODELS-2

Change-Id: I6094d0db688256ff45be75414f5d4271ae9e2234
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

3 files changed, 107 insertions, 64 deletions

diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh
index 2a3f8f2..d5568f4 100644
--- a/tools/kubernetes/demo_deploy.sh
+++ b/tools/kubernetes/demo_deploy.sh
@@ -18,8 +18,10 @@
 #. will be installed:
 #. - helm and dokuwiki as a demo helm chart based application
 #. - prometheus + grafana for cluster monitoring/stats
+#. And optionally, the following will be installed:
 #. - cloudify + kubernetes plugin and a demo hello world (nginx) app installed
 #. - OPNFV VES as an ONAP-compatible monitoring platform
+#. - Clearwater-docker as an example complex VNF
 #.
 #. Prerequisites:
 #. - MAAS server as cluster admin for k8s master/worker nodes.
@@ -28,7 +30,8 @@
 #. Usage: on the MAAS server
 #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
 #. $ bash ~/models/tools/kubernetes/demo_deploy.sh "<hosts>" <os> <key>
-#.   <master> "<workers>" <pub-net> <priv-net> <ceph-mode> "<ceph-dev>" [<extras>]
+#.   <master> "<workers>" <pub-net> <priv-net> <ceph-mode> "<ceph-dev>"
+#.   <base|all> [<extras>]
 #. <hosts>: space separated list of hostnames managed by MAAS
 #. <os>: OS to deploy, one of "ubuntu" (Xenial) or "centos" (Centos 7)
 #. <key>: name of private key for cluster node ssh (in current folder)
@@ -40,6 +43,7 @@
 #. <ceph-mode>: "helm" or "baremetal"
 #. <ceph-dev>: space-separated list of disks (e.g. sda, sdb) to use on each
 #.             worker, or folder (e.g. "/ceph")
+#. <base|all>: deploy k8s base services, or (for all) add Cloudify, VES, Clearwater
 #. <extras>: optional name of script for extra setup functions as needed
 #.
 #. See tools/demo_deploy.sh in the OPNFV VES repo for additional environment
@@ -64,6 +68,7 @@ function step_end() {
 }
 
 function run_master() {
+  trap 'fail' ERR
   start=$((`date +%s`/60))
   ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
     $k8s_user@$k8s_master <<EOF
@@ -76,7 +81,8 @@ EOF
 
 deploy_start=$((`date +%s`/60))
 
-extras=${10}
+deploy=${10}
+extras=${11}
 
 if [[ "$4" != "$5" ]]; then
   k8s_master_hostname=$(echo "$1" | cut -d ' ' -f 1)
@@ -116,8 +122,11 @@ step_end "source ~/models/tools/maas/deploy.sh $k8s_user $k8s_key \"$k8s_nodes\"
 
 eval `ssh-agent`
 ssh-add $k8s_key
-scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_key \
-  $k8s_user@$k8s_master:/home/$k8s_user/$k8s_key
+while ! scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_key \
+  $k8s_user@$k8s_master:/home/$k8s_user/$k8s_key ; do
+  echo; echo "$0 $(date): server is not yet ready for ssh; waiting 10 secs"
+  sleep 10
+done
 scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
   ~/k8s_env.sh $k8s_user@$k8s_master:/home/$k8s_user/k8s_env.sh
 
@@ -151,57 +160,63 @@ else
   echo; echo "$0 $(date): Skipping ceph (not yet working for AIO deployment)"
 fi
 
-echo; echo "$0 $(date): Setting up cloudify..."
-scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \
-  $k8s_user@$k8s_master:/home/$k8s_user/.
-run_master "bash cloudify/k8s-cloudify.sh prereqs"
-run_master "bash cloudify/k8s-cloudify.sh setup"
-
-echo; echo "$0 $(date): Verifying kubernetes+helm+ceph+cloudify install..."
-run "bash $HOME/models/tools/cloudify/k8s-cloudify.sh demo start"
-
-echo; echo "$0 $(date): Setting up VES..."
-# not re-cloned if existing - allows patch testing locally
-if [[ ! -d ~/ves ]]; then
-  echo; echo "$0 $(date): Cloning VES..."
-  git clone https://gerrit.opnfv.org/gerrit/ves ~/ves
-fi
-# Can't pass quoted strings in commands
-start=$((`date +%s`/60))
-bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify
-step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify"
-
 echo; echo "Setting up Prometheus..."
 scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \
   $k8s_user@$k8s_master:/home/$k8s_user/.
 run_master "bash prometheus-tools.sh setup prometheus helm"
-run_master "bash prometheus-tools.sh setup grafana helm $k8s_master:3000"
+run_master "bash prometheus-tools.sh setup grafana helm"
+
+if [[ "$deploy" == "all" ]]; then
+  echo; echo "$0 $(date): Setting up cloudify..."
+  scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \
+    $k8s_user@$k8s_master:/home/$k8s_user/.
+  run_master "bash cloudify/k8s-cloudify.sh prereqs"
+  run_master "bash cloudify/k8s-cloudify.sh setup"
+
+  echo; echo "$0 $(date): Verifying kubernetes+helm+ceph+cloudify install..."
+  run "bash $HOME/models/tools/cloudify/k8s-cloudify.sh demo start"
+
+  echo; echo "$0 $(date): Setting up VES..."
+  # not re-cloned if existing - allows patch testing locally
+  if [[ ! -d ~/ves ]]; then
+    echo; echo "$0 $(date): Cloning VES..."
+    git clone https://gerrit.opnfv.org/gerrit/ves ~/ves
+  fi
+  # Can't pass quoted strings in commands
+  start=$((`date +%s`/60))
+  bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify
+  step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify"
+
+  echo; echo "Installing clearwater-docker..."
+  run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh start $k8s_master blsaws latest"
 
-echo; echo "Installing clearwater-docker..."
-run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh start $k8s_master blsaws latest"
+  echo; echo "Waiting 5 minutes for clearwater IMS to be fully ready..."
+  sleep 300
 
-echo; echo "Waiting 5 minutes for clearwater IMS to be fully ready..."
-sleep 300
+  echo; echo "Run clearwater-live-test..."
+  run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh test $k8s_master"
+fi
 
-echo; echo "Run clearwater-live-test..."
-run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh test $k8s_master"
- 
 echo; echo "$0 $(date): All done!"
 deploy_end=$((`date +%s`/60))
 runtime=$((deploy_end-deploy_start))
 log "Deploy \"$1\" duration = $runtime minutes"
 
-source ~/ves/tools/ves_env.sh
-#echo "Prometheus UI is available at http://$k8s_master:30990"
-echo "InfluxDB API is available at http://$ves_influxdb_host:$ves_influxdb_port/query&db=veseventsdb&q=<string>"
-echo "Grafana dashboards are available at http://$ves_grafana_host:$ves_grafana_port (login as $ves_grafana_auth)"
-echo "Grafana API is available at http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/v1/query?query=<string>"
 echo "Kubernetes API is available at https://$k8s_master:6443/api/v1/"
-echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$k8s_master/api/v3.1/status"
-port=$(bash ~/models/tools/cloudify/k8s-cloudify.sh nodePort nginx)
-echo "Cloudify-deployed demo app nginx is available at http://$k8s_master:$port"
+echo "Prometheus UI is available at http://$k8s_master:30990"
 if [[ "$k8s_master" != "$k8s_workers" ]]; then
   export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki)
   export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
   echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/"
 fi
+
+if [[ "$deploy" == "all" ]]; then
+  source ~/ves/tools/ves_env.sh
+  echo "InfluxDB API is available at http://$ves_influxdb_host:$ves_influxdb_port/query&db=veseventsdb&q=<string>"
+  echo "Grafana dashboards are available at http://$ves_grafana_host:$ves_grafana_port (login as $ves_grafana_auth)"
+  echo "Grafana API is available at http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/v1/query?query=<string>"
+  echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$k8s_master/api/v3.1/status"
+  port=$(bash ~/models/tools/cloudify/k8s-cloudify.sh nodePort nginx)
+  echo "Cloudify-deployed demo app nginx is available at http://$k8s_master:$port"
+fi
+
diff --git a/tools/kubernetes/helm-tools.sh b/tools/kubernetes/helm-tools.sh
index a28b340..aa4bf23 100644
--- a/tools/kubernetes/helm-tools.sh
+++ b/tools/kubernetes/helm-tools.sh
@@ -39,13 +39,16 @@ function log() {
 function setup_helm() {
   log "Setup helm"
   # Install Helm
+  # per https://github.com/kubernetes/helm/blob/master/docs/install.md
   cd ~
   curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh
   chmod 700 get_helm.sh
   ./get_helm.sh
+  log "Initialize helm"
   helm init
-  nohup helm serve > /dev/null 2>&1 &
-  helm repo update
+#  nohup helm serve > /dev/null 2>&1 &
+#  log "Run helm repo update"
+#  helm repo update
   # TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224
   # For testing use only!
   kubectl create clusterrolebinding permissive-binding \
@@ -69,7 +72,8 @@ function setup_helm() {
 
 function wait_for_service() {
   log "Waiting for service $1 to be available"
-  pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
+  # TODO: fix 'head' workaround for more than one pod per service
+  pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }" | head -1)
   log "Service $1 is at pod $pod"
   ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
   while [[ "$ready" != "true" ]]; do
diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh
index e9293f6..9ff75fe 100644
--- a/tools/kubernetes/k8s-cluster.sh
+++ b/tools/kubernetes/k8s-cluster.sh
@@ -46,18 +46,21 @@
 #. Status: work in progress, incomplete
 #
 
-trap 'fail' ERR
+# TODO: Debug why some commands below will trigger fail incorrectly
+#  trap 'fail' ERR
 
-function fail() {
-  log $1
-  exit 1
-}
+#  function fail() {
+#    log $1
+#    exit 1
+#  }
 
 function log() {
   f=$(caller 0 | awk '{print $2}')
   l=$(caller 0 | awk '{print $1}')
   echo; echo "$f:$l ($(date)) $1"
-  kubectl get pods --all-namespaces
+  if [[ "$kubectl_status" == "ready" ]]; then
+    kubectl get pods --all-namespaces
+  fi
 }
 
 function setup_prereqs() {
@@ -85,15 +88,27 @@ if [[ "$dist" == "ubuntu" ]]; then
 
   wait_dpkg; sudo apt-get update
   wait_dpkg; sudo apt-get upgrade -y
-  echo; echo "prereqs.sh: ($(date)) Install latest docker"
-  wait_dpkg; sudo apt-get install -y docker.io
-  # Alternate for 1.12.6
-  #sudo apt-get install -y libltdl7
-  #wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
-  #sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
-  sudo service docker restart
+
+  dce=$(dpkg -l | grep -c docker-ce)
+  if [[ $dce -eq 0 ]]; then
+    echo; echo "prereqs.sh: ($(date)) Install latest docker-ce"
+    # Per https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/
+    sudo apt-get remove -y docker docker-engine docker.io docker-ce
+    sudo apt-get update
+    sudo apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    sudo add-apt-repository "deb [arch=amd64] \
+      https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    sudo apt-get update
+    sudo apt-get install -y docker-ce
+  fi
+
   echo; echo "prereqs.sh: ($(date)) Get k8s packages"
-  export KUBE_VERSION=1.7.5
+  export KUBE_VERSION=1.10.0
   # per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
   # Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/
   sudo apt-get update && sudo apt-get install -y apt-transport-https
@@ -179,8 +194,15 @@ function setup_k8s_master() {
   log "Reset kubeadm in case pre-existing cluster"
   sudo kubeadm reset
   # Start cluster
-  log "Start the cluster"
+  log "Workaround issue '/etc/kubernetes/manifests is not empty'"
 	mkdir ~/tmp
+  # workaround for [preflight] Some fatal errors occurred:
+	#                /etc/kubernetes/manifests is not empty
+  sudo rm -rf /etc/kubernetes/manifests/*
+  log "Disable swap to workaround k8s incompatibility with swap"
+  # per https://github.com/kubernetes/kubeadm/issues/610
+  sudo swapoff -a
+  log "Start the cluster"
   sudo kubeadm init --pod-network-cidr=192.168.0.0/16 >>~/tmp/kubeadm.out
   cat ~/tmp/kubeadm.out
   export k8s_joincmd=$(grep "kubeadm join" ~/tmp/kubeadm.out)
@@ -188,6 +210,7 @@ function setup_k8s_master() {
   mkdir -p $HOME/.kube
   sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
   sudo chown $(id -u):$(id -g) $HOME/.kube/config
+  export KUBECONFIG=$HOME/.kube/config
   # Deploy pod network
   log "Deploy calico as CNI"
   # Updated to deploy Calico 2.6 per the create-cluster-kubeadm guide above
@@ -198,7 +221,7 @@ function setup_k8s_master() {
   # Failure to wait for all calico pods to be running can cause the first worker
   # to be incompletely setup. Symptom is that node_ports cannot be routed
   # via that node (no response - incoming SYN packets are dropped). 
-  log "Wait for calico pods to be Running"
+  log "Wait for all calico pods to be Created"
   # calico-etcd, calico-kube-controllers, calico-node
   pods=$(kubectl get pods --namespace kube-system | grep -c calico)
   while [[ $pods -lt 3 ]]; do
@@ -207,23 +230,24 @@ function setup_k8s_master() {
     pods=$(kubectl get pods --namespace kube-system | grep -c calico)
   done
   
-  pods=$(kubectl get pods --all-namespaces | awk '/calico/ {print $2}')
+  log "Wait for all calico pods to be Running"
+  pods=$(kubectl get pods --namespace kube-system | awk '/calico/ {print $1}')
   for pod in $pods; do
-    status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}")
+    status=$(kubectl get pods --namespace kube-system | awk "/$pod/ {print \$3}")
     while [[ "$status" != "Running" ]]; do
       log "$pod status is $status. Waiting 10 seconds"
       sleep 10
-      status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}")
+      status=$(kubectl get pods --namespace kube-system | awk "/$pod/ {print \$3}")
     done
     log "$pod status is $status"
   done
 
   log "Wait for kubedns to be Running"
-  kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}')
+  kubedns=$(kubectl get pods --namespace kube-system | awk '/kube-dns/ {print $3}')
   while [[ "$kubedns" != "Running" ]]; do
     log "kube-dns status is $kubedns. Waiting 60 seconds"
     sleep 60
-    kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}')
+    kubedns=$(kubectl get pods --namespace kube-system | awk '/kube-dns/ {print $3}')
   done
   log "kube-dns status is $kubedns"