From 87f74be0cfe0de03bff32b67423f23fcf332e9e1 Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Fri, 24 Nov 2017 11:13:32 -0800 Subject: Factor out helm-tools.sh functions JIRA: MODELS-2 Change-Id: I4d2d145e13d441ff562183e9ed4ffeb951065db9 Signed-off-by: Bryan Sullivan --- tools/kubernetes/helm-tools.sh | 203 ++++++++++++++++++++++++++++++++++++++++ tools/kubernetes/k8s-cluster.sh | 170 +++------------------------------ 2 files changed, 216 insertions(+), 157 deletions(-) create mode 100644 tools/kubernetes/helm-tools.sh diff --git a/tools/kubernetes/helm-tools.sh b/tools/kubernetes/helm-tools.sh new file mode 100644 index 0000000..e528a15 --- /dev/null +++ b/tools/kubernetes/helm-tools.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# Copyright 2017 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#. What this is: script to setup Helm as kubernetes chart manager, and to deploy +#. demo apps. +#. Prerequisites: +#. - Kubernetes cluster deployed using k8s-cluster.sh (demo charts supported +#. leverage the ceph SDS storage classes setup by k8s-cluster.sh) +#. Usage: +# Intended to be called from k8s-cluster.sh. To run directly: +#. $ bash ceph-tools.sh setup +#. $ bash ceph-tools.sh +#. start|stop: start or stop the demo app +#. chart: name of helm chart; currently implemented charts include nginx, +#. mediawiki, dokuwiki, wordpress, redmine +#. For info see https://github.com/kubernetes/charts/tree/master/stable +#. +#. Status: work in progress, incomplete +# + +function log() { + f=$(caller 0 | awk '{print $2}') + l=$(caller 0 | awk '{print $1}') + echo "$f:$l ($(date)) $1" +} + +function setup_helm() { + log "Setup helm" + # Install Helm + cd ~ + curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh + chmod 700 get_helm.sh + ./get_helm.sh + helm init + nohup helm serve > /dev/null 2>&1 & + helm repo update + # TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224 + # For testing use only! + kubectl create clusterrolebinding permissive-binding \ + --clusterrole=cluster-admin --user=admin --user=kubelet \ + --group=system:serviceaccounts; + # TODO: workaround for tiller FailedScheduling (No nodes are available that + # match all of the following predicates:: PodToleratesNodeTaints (1).) + # kubectl taint nodes $HOSTNAME node-role.kubernetes.io/master:NoSchedule- + # Wait till tiller is running + tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}') + while [[ "$tiller_deploy" != "Running" ]]; do + log "tiller-deploy status is $tiller_deploy. Waiting 60 seconds for it to be 'Running'" + sleep 60 + tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}') + done + log "tiller-deploy status is $tiller_deploy" + + # Install services via helm charts from https://kubeapps.com/charts + # e.g. helm install stable/dokuwiki +} + +function wait_for_service() { + log "Waiting for service $1 to be available" + pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") + log "Service $1 is at pod $pod" + ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) + while [[ "$ready" != "true" ]]; do + log "pod $1 is not yet ready... waiting 10 seconds" + sleep 10 + # TODO: figure out why transient pods sometimes mess up this logic, thus need to re-get the pods + pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") + ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) + done + log "pod $pod is ready" + host_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.hostIP}' $pod) + port=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services $1) + log "$pod pod is running on assigned node $host_ip" + log "$1 service is assigned node_port $port" + log "verify $1 service is accessible via all workers at node_port $port" + nodes=$(kubectl get nodes | awk '/Ready/ {print $1}') + for node in $nodes; do + ip=$(kubectl describe nodes $node | awk '/InternalIP/ { print $2}') + while ! curl http://$ip:$port ; do + log "$1 service is not yet responding at worker $node IP $ip... waiting 10 seconds" + sleep 10 + done + log "$1 service is accessible at worker $node at http://$ip:$port" + done +} + +function start_chart() { + rm -rf /tmp/git/charts + git clone https://github.com/kubernetes/charts.git /tmp/git/charts + cd /tmp/git/charts/stable + case "$1" in + nginx) + rm -rf /tmp/git/helm + git clone https://github.com/kubernetes/helm.git /tmp/git/helm + cd /tmp/git/helm/docs/examples + sed -i -- 's/type: ClusterIP/type: NodePort/' ./nginx/values.yaml + helm install --name nx -f ./nginx/values.yaml ./nginx + wait_for_service nx-nginx + ;; + mediawiki) + mkdir ./mediawiki/charts + cp -r ./mariadb ./mediawiki/charts + # LoadBalancer is N/A for baremetal (public cloud only) - use NodePort + sed -i -- 's/LoadBalancer/NodePort/g' ./mediawiki/values.yaml + # Select the storageClass created in the ceph setup step + sed -i -- 's/# storageClass:/storageClass: "general"/g' ./mediawiki/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./mediawiki/charts/mariadb/values.yaml + helm install --name mw -f ./mediawiki/values.yaml ./mediawiki + wait_for_service mw-mediawiki + ;; + dokuwiki) + sed -i -- 's/# storageClass:/storageClass: "general"/g' ./dokuwiki/values.yaml + sed -i -- 's/LoadBalancer/NodePort/g' ./dokuwiki/values.yaml + helm install --name dw -f ./dokuwiki/values.yaml ./dokuwiki + wait_for_service dw-dokuwiki + ;; + wordpress) + mkdir ./wordpress/charts + cp -r ./mariadb ./wordpress/charts + sed -i -- 's/LoadBalancer/NodePort/g' ./wordpress/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/charts/mariadb/values.yaml + helm install --name wp -f ./wordpress/values.yaml ./wordpress + wait_for_service wp-wordpress + ;; + redmine) + mkdir ./redmine/charts + cp -r ./mariadb ./redmine/charts + cp -r ./postgresql ./redmine/charts + sed -i -- 's/LoadBalancer/NodePort/g' ./redmine/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/mariadb/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/postgresql/values.yaml + helm install --name rdm -f ./redmine/values.yaml ./redmine + wait_for_service rdm-redmine + ;; + owncloud) + # NOT YET WORKING: needs resolvable hostname for service + mkdir ./owncloud/charts + cp -r ./mariadb ./owncloud/charts + sed -i -- 's/LoadBalancer/NodePort/g' ./owncloud/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/charts/mariadb/values.yaml + helm install --name oc -f ./owncloud/values.yaml ./owncloud + wait_for_service oc-owncloud + ;; + *) + log "demo not implemented for $1" + esac +# extra useful commands +# kubectl describe pvc +# kubectl get pvc +# kubectl describe pods +# kubectl get pods --namespace default +# kubectl get pods --all-namespaces +# kubectl get svc --namespace default dw-dokuwiki +# kubectl describe svc --namespace default dw-dokuwiki +# kubectl describe pods --namespace default dw-dokuwiki +} + +function stop_chart() { + log "stop chart $1" + service=$(kubectl get services --namespace default | awk "/$1/ {print \$1}") + kubectl delete services --namespace default $service + secret=$(kubectl get secrets --namespace default | awk "/$1/ {print \$1}") + kubectl delete secrets --namespace default $secret + pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") + kubectl delete pods --namespace default $pod + release=$(echo $service | cut -d '-' -f 1) + helm del --purge $release + job=$(kubectl get jobs --namespace default | awk "/$1/ {print \$1}") + kubectl delete jobs --namespace default $job +} + +export WORK_DIR=$(pwd) +case "$1" in + setup) + setup_helm + ;; + start) + start_chart $2 + ;; + stop) + stop_chart $2 + ;; + clean) + # TODO + ;; + *) + if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then grep '#. ' $0; fi +esac diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index d89b2e5..0d351ba 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -24,6 +24,9 @@ #. $ bash k8s-cluster.sh master #. $ bash k8s-cluster.sh workers "" #. nodes: space-separated list of ceph node IPs +#. $ bash k8s-cluster.sh helm +#. Setup helm as kubernetes app management tool. Note this is a +#. prerequisite for selecting "helm" ceph-mode as described below. #. $ bash k8s-cluster.sh ceph "" [ceph_dev] #. nodes: space-separated list of ceph node IPs #. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24 @@ -31,12 +34,10 @@ #. ceph-mode: "helm" or "baremetal" #. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE*** #. if not provided, ceph data will be stored on osd nodes in /ceph -#. $ bash k8s-cluster.sh helm -#. Setup helm as app kubernetes orchestration tool -#. $ bash k8s-cluster.sh demo -#. Install helm charts for mediawiki and dokuwiki #. $ bash k8s-cluster.sh all "" [ceph_dev] -#. Runs all the steps above +#. Runs all the steps above, including starting dokuwiki demo app. +#. $ bash k8s-cluster.sh demo +#. Start or stop demo helm charts. See helm-tools.sh for chart options. #. #. When deployment is complete, the k8s API will be available at the master #. node, e.g. via: curl -k https://:6443/api/v1 @@ -167,151 +168,6 @@ function setup_k8s_workers() { log "Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'." } -function wait_for_service() { - log "Waiting for service $1 to be available" - pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") - log "Service $1 is at pod $pod" - ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) - while [[ "$ready" != "true" ]]; do - log "pod $1 is not yet ready... waiting 10 seconds" - sleep 10 - # TODO: figure out why transient pods sometimes mess up this logic, thus need to re-get the pods - pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") - ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) - done - log "pod $pod is ready" - host_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.hostIP}' $pod) - port=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services $1) - log "$pod pod is running on assigned node $host_ip" - log "$1 service is assigned node_port $port" - log "verify $1 service is accessible via all workers at node_port $port" - nodes=$(kubectl get nodes | awk '/Ready/ {print $1}') - for node in $nodes; do - ip=$(kubectl describe nodes $node | awk '/InternalIP/ { print $2}') - while ! curl http://$ip:$port ; do - log "$1 service is not yet responding at worker $node IP $ip... waiting 10 seconds" - sleep 10 - done - log "$1 service is accessible at worker $node at http://$ip:$port" - done -} - -function stop_chart() { - log "stop chart $1" - service=$(kubectl get services --namespace default | awk "/$1/ {print \$1}") - kubectl delete services --namespace default $service - secret=$(kubectl get secrets --namespace default | awk "/$1/ {print \$1}") - kubectl delete secrets --namespace default $secret - pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") - kubectl delete pods --namespace default $pod - release=$(echo $service | cut -d '-' -f 1) - helm del --purge $release - job=$(kubectl get jobs --namespace default | awk "/$1/ {print \$1}") - kubectl delete jobs --namespace default $job -} - -function start_chart() { - rm -rf /tmp/git/charts - git clone https://github.com/kubernetes/charts.git /tmp/git/charts - cd /tmp/git/charts/stable - case "$1" in - nginx) - rm -rf /tmp/git/helm - git clone https://github.com/kubernetes/helm.git /tmp/git/helm - cd /tmp/git/helm/docs/examples - sed -i -- 's/type: ClusterIP/type: NodePort/' ./nginx/values.yaml - helm install --name nx -f ./nginx/values.yaml ./nginx - wait_for_service nx-nginx - ;; - mediawiki) - mkdir ./mediawiki/charts - cp -r ./mariadb ./mediawiki/charts - # LoadBalancer is N/A for baremetal (public cloud only) - use NodePort - sed -i -- 's/LoadBalancer/NodePort/g' ./mediawiki/values.yaml - # Select the storageClass created in the ceph setup step - sed -i -- 's/# storageClass:/storageClass: "general"/g' ./mediawiki/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./mediawiki/charts/mariadb/values.yaml - helm install --name mw -f ./mediawiki/values.yaml ./mediawiki - wait_for_service mw-mediawiki - ;; - dokuwiki) - sed -i -- 's/# storageClass:/storageClass: "general"/g' ./dokuwiki/values.yaml - sed -i -- 's/LoadBalancer/NodePort/g' ./dokuwiki/values.yaml - helm install --name dw -f ./dokuwiki/values.yaml ./dokuwiki - wait_for_service dw-dokuwiki - ;; - wordpress) - mkdir ./wordpress/charts - cp -r ./mariadb ./wordpress/charts - sed -i -- 's/LoadBalancer/NodePort/g' ./wordpress/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./wordpress/charts/mariadb/values.yaml - helm install --name wp -f ./wordpress/values.yaml ./wordpress - wait_for_service wp-wordpress - ;; - redmine) - mkdir ./redmine/charts - cp -r ./mariadb ./redmine/charts - cp -r ./postgresql ./redmine/charts - sed -i -- 's/LoadBalancer/NodePort/g' ./redmine/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/mariadb/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./redmine/charts/postgresql/values.yaml - helm install --name rdm -f ./redmine/values.yaml ./redmine - wait_for_service rdm-redmine - ;; - owncloud) - # NOT YET WORKING: needs resolvable hostname for service - mkdir ./owncloud/charts - cp -r ./mariadb ./owncloud/charts - sed -i -- 's/LoadBalancer/NodePort/g' ./owncloud/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/values.yaml - sed -i -- 's/# storageClass: "-"/storageClass: "general"/g' ./owncloud/charts/mariadb/values.yaml - helm install --name oc -f ./owncloud/values.yaml ./owncloud - wait_for_service oc-owncloud - ;; - *) - log "demo not implemented for $1" - esac -# extra useful commands -# kubectl describe pvc -# kubectl get pvc -# kubectl describe pods -# kubectl get pods --namespace default -# kubectl get pods --all-namespaces -# kubectl get svc --namespace default dw-dokuwiki -# kubectl describe svc --namespace default dw-dokuwiki -# kubectl describe pods --namespace default dw-dokuwiki -} - -function setup_helm() { - log "Setup helm" - # Install Helm - cd ~ - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh - chmod 700 get_helm.sh - ./get_helm.sh - helm init - nohup helm serve > /dev/null 2>&1 & - helm repo update - # TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224 - # For testing use only! - kubectl create clusterrolebinding permissive-binding --clusterrole=cluster-admin --user=admin --user=kubelet --group=system:serviceaccounts; - # TODO: workaround for tiller FailedScheduling (No nodes are available that match all of the following predicates:: PodToleratesNodeTaints (1).) - # kubectl taint nodes $HOSTNAME node-role.kubernetes.io/master:NoSchedule- - # Wait till tiller is running - tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}') - while [[ "$tiller_deploy" != "Running" ]]; do - log "tiller-deploy status is $tiller_deploy. Waiting 60 seconds for it to be 'Running'" - sleep 60 - tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}') - done - log "tiller-deploy status is $tiller_deploy" - - # Install services via helm charts from https://kubeapps.com/charts - # e.g. helm install stable/dokuwiki -} - function setup_ceph() { if [[ "$4" == "helm" ]]; then source ./ceph-helm.sh "$1" $2 $3 $5 @@ -338,23 +194,23 @@ case "$1" in setup_ceph "$2" $3 $4 $5 $6 ;; helm) - setup_helm + bash ./helm-tools.sh setup ;; demo) if [[ "$2" == "start" ]]; then - start_chart $3 + bash ./helm-tools.sh start $3 else - stop_chart $3 + bash ./helm-tools.sh stop $3 fi ;; all) setup_k8s_master setup_k8s_workers "$2" - setup_helm - start_chart nginx - stop_chart nginx + bash ./helm-tools.sh setup + bash ./helm-tools.sh start nginx + bash ./helm-tools.sh stop nginx setup_ceph "$2" $3 $4 $5 $6 - start_chart dokuwiki + bash ./helm-tools.sh start dokuwiki ;; clean) # TODO -- cgit 1.2.3-korg