Factor out helm-tools.sh functions

JIRA: MODELS-2

Change-Id: I4d2d145e13d441ff562183e9ed4ffeb951065db9
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

2 files changed, 216 insertions, 157 deletions

diff --git a/tools/kubernetes/helm-tools.sh b/tools/kubernetes/helm-tools.sh
new file mode 100644
index 0000000..e528a15
--- /dev/null
+++ b/tools/kubernetes/helm-tools.sh
@@ -0,0 +1,203 @@
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: script to setup Helm as kubernetes chart manager, and to deploy
+#. demo apps.
+#. Prerequisites:
+#. - Kubernetes cluster deployed using k8s-cluster.sh (demo charts supported
+#.   leverage the ceph SDS storage classes setup by k8s-cluster.sh)
+#. Usage:
+#  Intended to be called from k8s-cluster.sh. To run directly:
+#. $ bash ceph-tools.sh setup
+#. $ bash ceph-tools.sh <start|stop> <chart>
+#.     start|stop: start or stop the demo app
+#.     chart: name of helm chart; currently implemented charts include nginx, 
+#.       mediawiki, dokuwiki, wordpress, redmine
+#.       For info see https://github.com/kubernetes/charts/tree/master/stable
+#.
+#. Status: work in progress, incomplete
+#
+
+function log() {
+  f=$(caller 0 | awk '{print $2}')
+  l=$(caller 0 | awk '{print $1}')
+  echo "$f:$l ($(date)) $1"
+}
+
+function setup_helm() {
+  log "Setup helm"
+  # Install Helm
+  cd ~
+  curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh
+  chmod 700 get_helm.sh
+  ./get_helm.sh
+  helm init
+  nohup helm serve > /dev/null 2>&1 &
+  helm repo update
+  # TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224
+  # For testing use only!
+  kubectl create clusterrolebinding permissive-binding \
+    --clusterrole=cluster-admin --user=admin --user=kubelet \
+    --group=system:serviceaccounts;
+  # TODO: workaround for tiller FailedScheduling (No nodes are available that 
+  # match all of the following predicates:: PodToleratesNodeTaints (1).)
+  # kubectl taint nodes $HOSTNAME node-role.kubernetes.io/master:NoSchedule-
+  # Wait till tiller is running
+  tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}')
+  while [[ "$tiller_deploy" != "Running" ]]; do
+    log "tiller-deploy status is $tiller_deploy. Waiting 60 seconds for it to be 'Running'"
+    sleep 60
+    tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}')
+  done
+  log "tiller-deploy status is $tiller_deploy"
+
+  # Install services via helm charts from https://kubeapps.com/charts
+  # e.g. helm install stable/dokuwiki
+}
+
+function wait_for_service() {
+  log "Waiting for service $1 to be available"
+  pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
+  log "Service $1 is at pod $pod"
+  ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
+  while [[ "$ready" != "true" ]]; do
+    log "pod $1 is not yet ready... waiting 10 seconds"
+    sleep 10
+    # TODO: figure out why transient pods sometimes mess up this logic, thus need to re-get the pods
+    pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
+    ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
+  done
+  log "pod $pod is ready"
+  host_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.hostIP}' $pod)
+  port=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services $1)
+  log "$pod pod is running on assigned node $host_ip"
+  log "$1 service is assigned node_port $port"
+  log "verify $1 service is accessible via all workers at node_port $port"
+  nodes=$(kubectl get nodes | awk '/Ready/ {print $1}')
+  for node in $nodes; do
+    ip=$(kubectl describe nodes $node | awk '/InternalIP/ { print $2}')
+    while ! curl http://$ip:$port ; do
+      log "$1 service is not yet responding at worker $node IP $ip... waiting 10 seconds"
+      sleep 10
+    done
+    log "$1 service is accessible at worker $node at http://$ip:$port"
+  done
+}
+
+function start_chart() {
+  rm -rf /tmp/git/charts
+  git clone https://github.com/kubernetes/charts.git /tmp/git/charts
+  cd /tmp/git/charts/stable
+  case "$1" in
+    nginx)
+      rm -rf /tmp/git/helm
+      git clone https://github.com/kubernetes/helm.git /tmp/git/helm
+      cd /tmp/git/helm/docs/examples
+      sed -i -- 's/type: ClusterIP/type: NodePort/' ./nginx/values.yaml
+      helm install --name nx -f ./nginx/values.yaml ./nginx
+      wait_for_service nx-nginx
+      ;;
+    mediawiki)
+      mkdir ./mediawiki/charts
+      cp -r ./mariadb ./mediawiki/charts
+      # LoadBalancer is N/A for baremetal (public cloud only) - use NodePort
+      sed -i -- 's/LoadBalancer/NodePort/g' ./mediawiki/values.yaml
+      # Select the storageClass created in the ceph setup step
+      sed -i -- 's/# storageClass:/storageClass: "general"/g' ./mediawiki/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./mediawiki/charts/mariadb/values.yaml
+      helm install --name mw -f ./mediawiki/values.yaml ./mediawiki
+      wait_for_service mw-mediawiki
+      ;;
+    dokuwiki)
+      sed -i -- 's/# storageClass:/storageClass: "general"/g' ./dokuwiki/values.yaml
+      sed -i -- 's/LoadBalancer/NodePort/g' ./dokuwiki/values.yaml
+      helm install --name dw -f ./dokuwiki/values.yaml ./dokuwiki
+      wait_for_service dw-dokuwiki
+      ;;
+    wordpress)
+      mkdir ./wordpress/charts
+      cp -r ./mariadb ./wordpress/charts
+      sed -i -- 's/LoadBalancer/NodePort/g' ./wordpress/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/charts/mariadb/values.yaml
+      helm install --name wp -f ./wordpress/values.yaml ./wordpress
+      wait_for_service wp-wordpress
+      ;;
+    redmine)
+      mkdir ./redmine/charts
+      cp -r ./mariadb ./redmine/charts
+      cp -r ./postgresql ./redmine/charts
+      sed -i -- 's/LoadBalancer/NodePort/g' ./redmine/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/mariadb/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/postgresql/values.yaml
+      helm install --name rdm -f ./redmine/values.yaml ./redmine
+      wait_for_service rdm-redmine
+      ;;
+    owncloud)
+      # NOT YET WORKING: needs resolvable hostname for service
+      mkdir ./owncloud/charts
+      cp -r ./mariadb ./owncloud/charts
+      sed -i -- 's/LoadBalancer/NodePort/g' ./owncloud/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/values.yaml
+      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/charts/mariadb/values.yaml
+      helm install --name oc -f ./owncloud/values.yaml ./owncloud
+      wait_for_service oc-owncloud
+      ;;
+    *)
+      log "demo not implemented for $1"
+  esac
+# extra useful commands
+# kubectl describe pvc
+# kubectl get pvc
+# kubectl describe pods
+# kubectl get pods --namespace default
+# kubectl get pods --all-namespaces
+# kubectl get svc --namespace default dw-dokuwiki
+# kubectl describe svc --namespace default dw-dokuwiki
+# kubectl describe pods --namespace default dw-dokuwiki
+}
+
+function stop_chart() {
+  log "stop chart $1"
+  service=$(kubectl get services --namespace default | awk "/$1/ {print \$1}")
+  kubectl delete services --namespace default $service
+  secret=$(kubectl get secrets --namespace default | awk "/$1/ {print \$1}")
+  kubectl delete secrets --namespace default $secret
+  pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
+  kubectl delete pods --namespace default $pod
+  release=$(echo $service | cut -d '-' -f 1)
+  helm del --purge $release
+  job=$(kubectl get jobs --namespace default | awk "/$1/ {print \$1}")
+  kubectl delete jobs --namespace default $job
+}
+
+export WORK_DIR=$(pwd)
+case "$1" in
+  setup)
+    setup_helm
+    ;;
+  start)
+    start_chart $2
+    ;;
+  stop)
+    stop_chart $2
+    ;;
+  clean)
+    # TODO
+    ;;
+  *)
+    if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then grep '#. ' $0; fi
+esac
diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh
index d89b2e5..0d351ba 100644
--- a/tools/kubernetes/k8s-cluster.sh
+++ b/tools/kubernetes/k8s-cluster.sh
@@ -24,6 +24,9 @@
 #. $ bash k8s-cluster.sh master
 #. $ bash k8s-cluster.sh workers "<nodes>"
 #.     nodes: space-separated list of ceph node IPs
+#. $ bash k8s-cluster.sh helm
+#.     Setup helm as kubernetes app management tool. Note this is a
+#.     prerequisite for selecting "helm" ceph-mode as described below.
 #. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev]
 #.     nodes: space-separated list of ceph node IPs
 #.     cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24
@@ -31,12 +34,10 @@
 #.     ceph-mode: "helm" or "baremetal"
 #.     ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE***
 #.               if not provided, ceph data will be stored on osd nodes in /ceph
-#. $ bash k8s-cluster.sh helm
-#.     Setup helm as app kubernetes orchestration tool
-#. $ bash k8s-cluster.sh demo
-#.     Install helm charts for mediawiki and dokuwiki
 #. $ bash k8s-cluster.sh all "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev]
-#.     Runs all the steps above
+#.     Runs all the steps above, including starting dokuwiki demo app.
+#. $ bash k8s-cluster.sh demo <start|stop> <chart>
+#.     Start or stop demo helm charts. See helm-tools.sh for chart options.
 #.
 #. When deployment is complete, the k8s API will be available at the master
 #. node, e.g. via: curl -k https://<master-ip>:6443/api/v1
@@ -167,151 +168,6 @@ function setup_k8s_workers() {
   log "Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'."
 }
 
-function wait_for_service() {
-  log "Waiting for service $1 to be available"
-  pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
-  log "Service $1 is at pod $pod"
-  ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
-  while [[ "$ready" != "true" ]]; do
-    log "pod $1 is not yet ready... waiting 10 seconds"
-    sleep 10
-    # TODO: figure out why transient pods sometimes mess up this logic, thus need to re-get the pods
-    pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
-    ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
-  done
-  log "pod $pod is ready"
-  host_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.hostIP}' $pod)
-  port=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services $1)
-  log "$pod pod is running on assigned node $host_ip"
-  log "$1 service is assigned node_port $port"
-  log "verify $1 service is accessible via all workers at node_port $port"
-  nodes=$(kubectl get nodes | awk '/Ready/ {print $1}')
-  for node in $nodes; do
-    ip=$(kubectl describe nodes $node | awk '/InternalIP/ { print $2}')
-    while ! curl http://$ip:$port ; do
-      log "$1 service is not yet responding at worker $node IP $ip... waiting 10 seconds"
-      sleep 10
-    done
-    log "$1 service is accessible at worker $node at http://$ip:$port"
-  done
-}
-
-function stop_chart() {
-  log "stop chart $1"
-  service=$(kubectl get services --namespace default | awk "/$1/ {print \$1}")
-  kubectl delete services --namespace default $service
-  secret=$(kubectl get secrets --namespace default | awk "/$1/ {print \$1}")
-  kubectl delete secrets --namespace default $secret
-  pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
-  kubectl delete pods --namespace default $pod
-  release=$(echo $service | cut -d '-' -f 1)
-  helm del --purge $release
-  job=$(kubectl get jobs --namespace default | awk "/$1/ {print \$1}")
-  kubectl delete jobs --namespace default $job
-}
-
-function start_chart() {
-  rm -rf /tmp/git/charts
-  git clone https://github.com/kubernetes/charts.git /tmp/git/charts
-  cd /tmp/git/charts/stable
-  case "$1" in
-    nginx)
-      rm -rf /tmp/git/helm
-      git clone https://github.com/kubernetes/helm.git /tmp/git/helm
-      cd /tmp/git/helm/docs/examples
-      sed -i -- 's/type: ClusterIP/type: NodePort/' ./nginx/values.yaml
-      helm install --name nx -f ./nginx/values.yaml ./nginx
-      wait_for_service nx-nginx
-      ;;
-    mediawiki)
-      mkdir ./mediawiki/charts
-      cp -r ./mariadb ./mediawiki/charts
-      # LoadBalancer is N/A for baremetal (public cloud only) - use NodePort
-      sed -i -- 's/LoadBalancer/NodePort/g' ./mediawiki/values.yaml
-      # Select the storageClass created in the ceph setup step
-      sed -i -- 's/# storageClass:/storageClass: "general"/g' ./mediawiki/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./mediawiki/charts/mariadb/values.yaml
-      helm install --name mw -f ./mediawiki/values.yaml ./mediawiki
-      wait_for_service mw-mediawiki
-      ;;
-    dokuwiki)
-      sed -i -- 's/# storageClass:/storageClass: "general"/g' ./dokuwiki/values.yaml
-      sed -i -- 's/LoadBalancer/NodePort/g' ./dokuwiki/values.yaml
-      helm install --name dw -f ./dokuwiki/values.yaml ./dokuwiki
-      wait_for_service dw-dokuwiki
-      ;;
-    wordpress)
-      mkdir ./wordpress/charts
-      cp -r ./mariadb ./wordpress/charts
-      sed -i -- 's/LoadBalancer/NodePort/g' ./wordpress/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/charts/mariadb/values.yaml
-      helm install --name wp -f ./wordpress/values.yaml ./wordpress
-      wait_for_service wp-wordpress
-      ;;
-    redmine)
-      mkdir ./redmine/charts
-      cp -r ./mariadb ./redmine/charts
-      cp -r ./postgresql ./redmine/charts
-      sed -i -- 's/LoadBalancer/NodePort/g' ./redmine/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/mariadb/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/postgresql/values.yaml
-      helm install --name rdm -f ./redmine/values.yaml ./redmine
-      wait_for_service rdm-redmine
-      ;;
-    owncloud)
-      # NOT YET WORKING: needs resolvable hostname for service
-      mkdir ./owncloud/charts
-      cp -r ./mariadb ./owncloud/charts
-      sed -i -- 's/LoadBalancer/NodePort/g' ./owncloud/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/values.yaml
-      sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/charts/mariadb/values.yaml
-      helm install --name oc -f ./owncloud/values.yaml ./owncloud
-      wait_for_service oc-owncloud
-      ;;
-    *)
-      log "demo not implemented for $1"
-  esac
-# extra useful commands
-# kubectl describe pvc
-# kubectl get pvc
-# kubectl describe pods
-# kubectl get pods --namespace default
-# kubectl get pods --all-namespaces
-# kubectl get svc --namespace default dw-dokuwiki
-# kubectl describe svc --namespace default dw-dokuwiki
-# kubectl describe pods --namespace default dw-dokuwiki
-}
-
-function setup_helm() {
-  log "Setup helm"
-  # Install Helm
-  cd ~
-  curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh
-  chmod 700 get_helm.sh
-  ./get_helm.sh
-  helm init
-  nohup helm serve > /dev/null 2>&1 &
-  helm repo update
-  # TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224
-  # For testing use only!
-  kubectl create clusterrolebinding permissive-binding --clusterrole=cluster-admin --user=admin --user=kubelet --group=system:serviceaccounts;
-  # TODO: workaround for tiller FailedScheduling (No nodes are available that match all of the following predicates:: PodToleratesNodeTaints (1).)
-  # kubectl taint nodes $HOSTNAME node-role.kubernetes.io/master:NoSchedule-
-  # Wait till tiller is running
-  tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}')
-  while [[ "$tiller_deploy" != "Running" ]]; do
-    log "tiller-deploy status is $tiller_deploy. Waiting 60 seconds for it to be 'Running'"
-    sleep 60
-    tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}')
-  done
-  log "tiller-deploy status is $tiller_deploy"
-
-  # Install services via helm charts from https://kubeapps.com/charts
-  # e.g. helm install stable/dokuwiki
-}
-
 function setup_ceph() {
   if [[ "$4" == "helm" ]]; then
     source ./ceph-helm.sh "$1" $2 $3 $5
@@ -338,23 +194,23 @@ case "$1" in
     setup_ceph "$2" $3 $4 $5 $6
     ;;
   helm)
-    setup_helm
+    bash ./helm-tools.sh setup
     ;;
   demo)
     if [[ "$2" == "start" ]]; then
-      start_chart $3
+      bash ./helm-tools.sh start $3
     else
-      stop_chart $3
+      bash ./helm-tools.sh stop $3
     fi
     ;;
   all)
     setup_k8s_master
     setup_k8s_workers "$2"
-    setup_helm
-    start_chart nginx
-    stop_chart nginx
+    bash ./helm-tools.sh setup
+    bash ./helm-tools.sh start nginx
+    bash ./helm-tools.sh stop nginx
     setup_ceph "$2" $3 $4 $5 $6
-    start_chart dokuwiki
+    bash ./helm-tools.sh start dokuwiki
     ;;
   clean)
     # TODO