Refactor demo deploy; fix issues with cluster firewall

JIRA: MODELS-2

Change-Id: I0511c8af02e5ed706ce943a27d74e5eb22637e58
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

3 files changed, 156 insertions, 70 deletions

diff --git a/tools/cloudify/k8s-cloudify.sh b/tools/cloudify/k8s-cloudify.sh
index 36abb63..8f52f88 100644
--- a/tools/cloudify/k8s-cloudify.sh
+++ b/tools/cloudify/k8s-cloudify.sh
@@ -9,7 +9,6 @@
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
@@ -19,6 +18,7 @@
 #. Usage:
 #.   From a server with access to the kubernetes master node:
 #.   $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
+#.   $ cd models/tools/cloudify
 #.   $ scp -r ~/models/tools/cloudify ubuntu@<k8s-master>:/home/ubuntu/.
 #.     <k8s-master>: IP or hostname of kubernetes master server
 #.   $ ssh -x ubuntu@<k8s-master> cloudify/k8s-cloudify.sh prereqs
@@ -51,8 +51,7 @@ function fail() {
 function log() {
   f=$(caller 0 | awk '{print $2}')
   l=$(caller 0 | awk '{print $1}')
-  echo ""
-  echo "$f:$l ($(date)) $1"
+  echo; echo "$f:$l ($(date)) $1"
 }
 
 function prereqs() {
@@ -178,7 +177,7 @@ function service_port() {
     sleep 10
     ((tries--))
   done
-  if [[ "$port" == "" ]]; then
+  if [[ "$port" == "null" ]]; then
     jq -r '.items' /tmp/json
     fail "node_port not found for service"
   fi
@@ -283,6 +282,7 @@ function stop() {
     -d "{\"deployment_id\":\"$bp\", \"workflow_id\":\"uninstall\"}" \
     -o /tmp/json http://$manager_ip/api/v3.1/executions
   id=$(jq -r ".id" /tmp/json)
+  log "uninstall execution id = $id"
   status=""
   tries=1
   while [[ "$status" != "terminated" && $tries -lt 10 ]]; do
diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh
index 7922434..c585a3e 100644
--- a/tools/kubernetes/demo_deploy.sh
+++ b/tools/kubernetes/demo_deploy.sh
@@ -34,7 +34,7 @@
 #. <key>: name of private key for cluster node ssh (in current folder)
 #. <hosts>: space separated list of hostnames managed by MAAS
 #. <master>: IP of cluster master node
-#. <workers>: space separated list of agent node IPs
+#. <workers>: space separated list of worker node IPs
 #. <pub-net>: CID formatted public network
 #. <priv-net>: CIDR formatted private network (may be same as pub-net)
 #. <ceph-mode>: "helm" or "baremetal"
@@ -51,43 +51,56 @@ ceph_mode=$7
 ceph_dev=$8
 extras=$9
 
+function run_master() {
+ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+  ubuntu@$master <<EOF
+exec ssh-agent bash
+ssh-add $key
+$1
+EOF
+}
+
 source ~/models/tools/maas/deploy.sh $1 "$2" $9
 eval `ssh-agent`
 ssh-add $key
-if [[ "x$extras" != "x" ]]; then source $extras; fi
 scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $key ubuntu@$master:/home/ubuntu/$key
-echo "$0 $(date): Setting up kubernetes..."
+
+echo; echo "$0 $(date): Setting up kubernetes master..."
 scp -r -o StrictHostKeyChecking=no ~/models/tools/kubernetes/* \
   ubuntu@$master:/home/ubuntu/.
-ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master <<EOF
-exec ssh-agent bash
-ssh-add $key
-bash k8s-cluster.sh all "$workers" $priv_net $pub_net $ceph_mode $ceph_dev
-EOF
-# TODO: Figure this out... Have to break the setup into two steps as something
-# causes the ssh session to end before the prometheus setup, if both scripts
-# are in the same ssh session
-echo "Setting up Prometheus..."
-ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master mkdir -p \
-  /home/ubuntu/models/tools/prometheus
+run_master "bash k8s-cluster.sh master"
+
+echo; echo "$0 $(date): Setting up kubernetes workers..."
+run_master "bash k8s-cluster.sh workers \"$workers\""
+
+echo; echo "$0 $(date): Setting up helm..."
+run_master "bash k8s-cluster.sh helm"
+
+echo; echo "$0 $(date): Verifying kubernetes+helm install..."
+run_master "bash k8s-cluster.sh demo start nginx"
+run_master "bash k8s-cluster.sh demo stop nginx"
+
+echo; echo "$0 $(date): Setting up ceph-helm"
+run_master "bash k8s-cluster.sh ceph \"$workers\" $priv_net $pub_net $ceph_mode $ceph_dev"
+
+echo; echo "$0 $(date): Verifying kubernetes+helm+ceph install..."
+run_master "bash k8s-cluster.sh demo start dokuwiki"
+
+echo; echo "Setting up Prometheus..."
 scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \
-  ubuntu@$master:/home/ubuntu/models/tools/prometheus
-ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master <<EOF
-exec ssh-agent bash
-ssh-add $key
-cd models/tools/prometheus
-bash prometheus-tools.sh all "$workers"
-EOF
-echo "$0 $(date): Setting up cloudify..."
+  ubuntu@$master:/home/ubuntu/.
+run_master "bash prometheus-tools.sh all \"$workers\""
+
+echo; echo "$0 $(date): Setting up cloudify..."
 scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \
   ubuntu@$master:/home/ubuntu/.
-ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master \
-  bash cloudify/k8s-cloudify.sh prereqs
-ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master \
-  bash cloudify/k8s-cloudify.sh setup
-source ~/models/tools/cloudify/k8s-cloudify.sh demo start $master
+run_master "bash cloudify/k8s-cloudify.sh prereqs"
+run_master "bash cloudify/k8s-cloudify.sh setup"
+
+echo; echo "$0 $(date): Verifying kubernetes+helm+ceph+cloudify install..."
+bash ~/models/tools/cloudify/k8s-cloudify.sh demo start $master
 
-echo "$0 $(date): All done!"
+echo; echo "$0 $(date): All done!"
 export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki)
 export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master  kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
 echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/"
diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh
index 54c5973..817caaf 100644
--- a/tools/kubernetes/k8s-cluster.sh
+++ b/tools/kubernetes/k8s-cluster.sh
@@ -15,14 +15,14 @@
 #
 #. What this is: script to setup a kubernetes cluster with calico as sni
 #. Prerequisites:
-#. - Ubuntu xenial server for master and agent nodes
-#. - key-based auth setup for ssh/scp between master and agent nodes
+#. - Ubuntu xenial server for master and worker nodes
+#. - key-based auth setup for ssh/scp between master and worker nodes
 #. - 192.168.0.0/16 should not be used on your server network interface subnets
 #. Usage:
 #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
 #. $ cd ~/models/tools/kubernetes
 #. $ bash k8s-cluster.sh master
-#. $ bash k8s-cluster.sh agents "<nodes>"
+#. $ bash k8s-cluster.sh workers "<nodes>"
 #.     nodes: space-separated list of ceph node IPs
 #. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev]
 #.     nodes: space-separated list of ceph node IPs
@@ -44,10 +44,15 @@
 #. Status: work in progress, incomplete
 #
 
+function fail() {
+  log $1
+  exit 1
+}
+
 function log() {
   f=$(caller 0 | awk '{print $2}')
   l=$(caller 0 | awk '{print $1}')
-  echo "$f:$l ($(date)) $1"
+  echo; echo "$f:$l ($(date)) $1"
 }
 
 function setup_prereqs() {
@@ -55,16 +60,21 @@ function setup_prereqs() {
   cat <<'EOG' >/tmp/prereqs.sh
 #!/bin/bash
 # Basic server pre-reqs
-sudo apt-get -y remove kubectl kubelet kubeadm
+echo; echo "prereqs.sh: ($(date)) Basic prerequisites"
 sudo apt-get update
 sudo apt-get upgrade -y
-# Set hostname
 if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then
+  echo; echo "prereqs.sh: ($(date)) Add $HOSTNAME to /etc/hosts"
   echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" | sudo tee -a /etc/hosts
 fi
-# Install docker 1.12 (default for xenial is 1.12.6)
+echo; echo "prereqs.sh: ($(date)) Install latest docker"
 sudo apt-get install -y docker.io
-sudo service docker start
+# Alternate for 1.12.6
+#sudo apt-get install -y libltdl7
+#wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
+#sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
+sudo service docker restart
+echo; echo "prereqs.sh: ($(date)) Get k8s packages"
 export KUBE_VERSION=1.7.5
 # per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
 # Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/
@@ -74,11 +84,31 @@ cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
 deb http://apt.kubernetes.io/ kubernetes-xenial main
 EOF
 sudo apt-get update
-# Next command is to workaround bug resulting in "PersistentVolumeClaim is not bound" for pod startup (remain in Pending)
-# TODO: reverify if this is still an issue in the final working script
+echo; echo "prereqs.sh: ($(date)) Install kubectl, kubelet, kubeadm"
 sudo apt-get -y install --allow-downgrades kubectl=${KUBE_VERSION}-00 kubelet=${KUBE_VERSION}-00 kubeadm=${KUBE_VERSION}-00
-# Needed for API output parsing
+echo; echo "prereqs.sh: ($(date)) Install jq for API output parsing"
 sudo apt-get -y install jq
+echo; echo "prereqs.sh: ($(date)) Set firewall rules"
+# Per https://kubernetes.io/docs/setup/independent/install-kubeadm/
+if [[ "$(sudo ufw status)" == "Status: active" ]]; then
+  if [[ "$1" == "master" ]]; then
+    sudo ufw allow 6443/tcp
+    sudo ufw allow 2379:2380/tcp
+    sudo ufw allow 10250/tcp
+    sudo ufw allow 10251/tcp
+    sudo ufw allow 10252/tcp
+    sudo ufw allow 10255/tcp
+  else
+    sudo ufw allow 10250/tcp
+    sudo ufw allow 10255/tcp
+    sudo ufw allow 30000:32767/tcp
+  fi
+fi
+# TODO: fix need for this workaround: disable firewall since the commands 
+# above do not appear to open the needed ports, even if ufw is inactive
+# (symptom: nodeport requests fail unless sent from within the cluster or
+# to the node IP where the pod is assigned) issue discovered ~11/16/17
+sudo ufw disable
 EOG
 }
 
@@ -103,13 +133,15 @@ function setup_k8s_master() {
   sudo chown $(id -u):$(id -g) $HOME/.kube/config
   # Deploy pod network
   log "Deploy calico as CNI"
-  sudo kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
+  # Updated to deploy Calico 2.6 per the create-cluster-kubeadm guide above
+  #  sudo kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
+  sudo kubectl apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
 }
 
-function setup_k8s_agents() {
-  agents="$1"
+function setup_k8s_workers() {
+  workers="$1"
   export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out)
-  log "Installing agents at $1 with joincmd: $k8s_joincmd"
+  log "Installing workers at $1 with joincmd: $k8s_joincmd"
 
   setup_prereqs
 
@@ -121,13 +153,15 @@ function setup_k8s_agents() {
   done
   log "kube-dns status is $kubedns"
 
-  for agent in $agents; do
-    log "Install agent at $agent"
-    scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/prereqs.sh ubuntu@$agent:/tmp/prereqs.sh
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent bash /tmp/prereqs.sh agent
+  for worker in $workers; do
+    log "Install worker at $worker"
+    if ! scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/prereqs.sh ubuntu@$worker:/tmp/prereqs.sh ; then
+      fail "Failed copying setup files to $worker"
+    fi
+    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker bash /tmp/prereqs.sh worker
     # Workaround for "[preflight] Some fatal errors occurred: /var/lib/kubelet is not empty" per https://github.com/kubernetes/kubeadm/issues/1
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent sudo kubeadm reset
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent sudo $k8s_joincmd
+    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker sudo kubeadm reset
+    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker sudo $k8s_joincmd
   done
 
   log "Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'."
@@ -139,29 +173,56 @@ function wait_for_service() {
   log "Service $1 is at pod $pod"
   ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
   while [[ "$ready" != "true" ]]; do
-    log "$1 container is not yet ready... waiting 10 seconds"
+    log "pod $1 is not yet ready... waiting 10 seconds"
     sleep 10
     # TODO: figure out why transient pods sometimes mess up this logic, thus need to re-get the pods
     pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
     ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
   done
-  log "pod $pod container status is $ready"
+  log "pod $pod is ready"
   host_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.hostIP}' $pod)
   port=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services $1)
-  log "pod $pod container is at host $host_ip and port $port"
-  while ! curl http://$host_ip:$port ; do
-    log "$1 service is not yet responding... waiting 10 seconds"
-    sleep 10
+  log "$pod pod is running on assigned node $host_ip"
+  log "$1 service is assigned node_port $port"
+  log "verify $1 service is accessible via all workers at node_port $port"
+  nodes=$(kubectl get nodes | awk '/Ready/ {print $1}')
+  for node in $nodes; do
+    ip=$(kubectl describe nodes $node | awk '/InternalIP/ { print $2}')
+    while ! curl http://$ip:$port ; do
+      log "$1 service is not yet responding at worker $node IP $ip... waiting 10 seconds"
+      sleep 10
+    done
+    log "$1 service is accessible at worker $node at http://$ip:$port"
   done
-  log "$1 is available at http://$host_ip:$port"
 }
 
-function demo_chart() {
-  cd ~
-  rm -rf charts
-  git clone https://github.com/kubernetes/charts.git
-  cd charts/stable
+function stop_chart() {
+  log "stop chart $1"
+  service=$(kubectl get services --namespace default | awk "/$1/ {print \$1}")
+  kubectl delete services --namespace default $service
+  secret=$(kubectl get secrets --namespace default | awk "/$1/ {print \$1}")
+  kubectl delete secrets --namespace default $secret
+  pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
+  kubectl delete pods --namespace default $pod
+  release=$(echo $service | cut -d '-' -f 1)
+  helm del --purge $release
+  job=$(kubectl get jobs --namespace default | awk "/$1/ {print \$1}")
+  kubectl delete jobs --namespace default $job
+}
+
+function start_chart() {
+  rm -rf /tmp/git/charts
+  git clone https://github.com/kubernetes/charts.git /tmp/git/charts
+  cd /tmp/git/charts/stable
   case "$1" in
+    nginx)
+      rm -rf /tmp/git/helm
+      git clone https://github.com/kubernetes/helm.git /tmp/git/helm
+      cd /tmp/git/helm/docs/examples
+      sed -i -- 's/type: ClusterIP/type: NodePort/' ./nginx/values.yaml
+      helm install --name nx -f ./nginx/values.yaml ./nginx
+      wait_for_service nx-nginx
+      ;;
     mediawiki)
       # NOT YET WORKING
       # mariadb: Readiness probe failed: mysqladmin: connect to server at 'localhost' failed
@@ -265,13 +326,19 @@ function setup_ceph() {
   fi
 }
 
+workers="$2"
+privnet=$3
+pubnet=$4
+ceph_mode=$5
+ceph_dev=$6
+
 export WORK_DIR=$(pwd)
 case "$1" in
   master)
     setup_k8s_master
     ;;
-  agents)
-    setup_k8s_agents "$2"
+  workers)
+    setup_k8s_workers "$2"
     ;;
   ceph)
     setup_ceph "$2" $3 $4 $5 $6
@@ -280,14 +347,20 @@ case "$1" in
     setup_helm
     ;;
   demo)
-    demo_chart $2
+    if [[ "$2" == "start" ]]; then
+      start_chart $3
+    else
+      stop_chart $3
+    fi
     ;;
   all)
     setup_k8s_master
-    setup_k8s_agents "$2"
+    setup_k8s_workers "$2"
     setup_helm
+    start_chart nginx
+    stop_chart nginx
     setup_ceph "$2" $3 $4 $5 $6
-    demo_chart dokuwiki
+    start_chart dokuwiki
     ;;
   clean)
     # TODO