From d40f36a0eecc24566b441d7e9298303defc4613f Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Mon, 11 Jun 2018 09:32:01 -0700 Subject: Update k8s and docker-ce versions Add option to deploy k8s core only JIRA: MODELS-2 Change-Id: I6094d0db688256ff45be75414f5d4271ae9e2234 Signed-off-by: Bryan Sullivan --- tools/kubernetes/demo_deploy.sh | 95 +++++++++++++++++++++--------------- tools/kubernetes/helm-tools.sh | 10 ++-- tools/kubernetes/k8s-cluster.sh | 66 +++++++++++++++++-------- tools/prometheus/prometheus-tools.sh | 16 +++--- 4 files changed, 116 insertions(+), 71 deletions(-) (limited to 'tools') diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh index 2a3f8f2..d5568f4 100644 --- a/tools/kubernetes/demo_deploy.sh +++ b/tools/kubernetes/demo_deploy.sh @@ -18,8 +18,10 @@ #. will be installed: #. - helm and dokuwiki as a demo helm chart based application #. - prometheus + grafana for cluster monitoring/stats +#. And optionally, the following will be installed: #. - cloudify + kubernetes plugin and a demo hello world (nginx) app installed #. - OPNFV VES as an ONAP-compatible monitoring platform +#. - Clearwater-docker as an example complex VNF #. #. Prerequisites: #. - MAAS server as cluster admin for k8s master/worker nodes. @@ -28,7 +30,8 @@ #. Usage: on the MAAS server #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models #. $ bash ~/models/tools/kubernetes/demo_deploy.sh "" -#. "" "" [] +#. "" "" +#. [] #. : space separated list of hostnames managed by MAAS #. : OS to deploy, one of "ubuntu" (Xenial) or "centos" (Centos 7) #. : name of private key for cluster node ssh (in current folder) @@ -40,6 +43,7 @@ #. : "helm" or "baremetal" #. : space-separated list of disks (e.g. sda, sdb) to use on each #. worker, or folder (e.g. "/ceph") +#. : deploy k8s base services, or (for all) add Cloudify, VES, Clearwater #. : optional name of script for extra setup functions as needed #. #. See tools/demo_deploy.sh in the OPNFV VES repo for additional environment @@ -64,6 +68,7 @@ function step_end() { } function run_master() { + trap 'fail' ERR start=$((`date +%s`/60)) ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ $k8s_user@$k8s_master <" -echo "Grafana dashboards are available at http://$ves_grafana_host:$ves_grafana_port (login as $ves_grafana_auth)" -echo "Grafana API is available at http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/v1/query?query=" echo "Kubernetes API is available at https://$k8s_master:6443/api/v1/" -echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$k8s_master/api/v3.1/status" -port=$(bash ~/models/tools/cloudify/k8s-cloudify.sh nodePort nginx) -echo "Cloudify-deployed demo app nginx is available at http://$k8s_master:$port" +echo "Prometheus UI is available at http://$k8s_master:30990" if [[ "$k8s_master" != "$k8s_workers" ]]; then export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki) export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/" fi + +if [[ "$deploy" == "all" ]]; then + source ~/ves/tools/ves_env.sh + echo "InfluxDB API is available at http://$ves_influxdb_host:$ves_influxdb_port/query&db=veseventsdb&q=" + echo "Grafana dashboards are available at http://$ves_grafana_host:$ves_grafana_port (login as $ves_grafana_auth)" + echo "Grafana API is available at http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/v1/query?query=" + echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$k8s_master/api/v3.1/status" + port=$(bash ~/models/tools/cloudify/k8s-cloudify.sh nodePort nginx) + echo "Cloudify-deployed demo app nginx is available at http://$k8s_master:$port" +fi + diff --git a/tools/kubernetes/helm-tools.sh b/tools/kubernetes/helm-tools.sh index a28b340..aa4bf23 100644 --- a/tools/kubernetes/helm-tools.sh +++ b/tools/kubernetes/helm-tools.sh @@ -39,13 +39,16 @@ function log() { function setup_helm() { log "Setup helm" # Install Helm + # per https://github.com/kubernetes/helm/blob/master/docs/install.md cd ~ curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh chmod 700 get_helm.sh ./get_helm.sh + log "Initialize helm" helm init - nohup helm serve > /dev/null 2>&1 & - helm repo update +# nohup helm serve > /dev/null 2>&1 & +# log "Run helm repo update" +# helm repo update # TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224 # For testing use only! kubectl create clusterrolebinding permissive-binding \ @@ -69,7 +72,8 @@ function setup_helm() { function wait_for_service() { log "Waiting for service $1 to be available" - pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") + # TODO: fix 'head' workaround for more than one pod per service + pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }" | head -1) log "Service $1 is at pod $pod" ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) while [[ "$ready" != "true" ]]; do diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index e9293f6..9ff75fe 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -46,18 +46,21 @@ #. Status: work in progress, incomplete # -trap 'fail' ERR +# TODO: Debug why some commands below will trigger fail incorrectly +# trap 'fail' ERR -function fail() { - log $1 - exit 1 -} +# function fail() { +# log $1 +# exit 1 +# } function log() { f=$(caller 0 | awk '{print $2}') l=$(caller 0 | awk '{print $1}') echo; echo "$f:$l ($(date)) $1" - kubectl get pods --all-namespaces + if [[ "$kubectl_status" == "ready" ]]; then + kubectl get pods --all-namespaces + fi } function setup_prereqs() { @@ -85,15 +88,27 @@ if [[ "$dist" == "ubuntu" ]]; then wait_dpkg; sudo apt-get update wait_dpkg; sudo apt-get upgrade -y - echo; echo "prereqs.sh: ($(date)) Install latest docker" - wait_dpkg; sudo apt-get install -y docker.io - # Alternate for 1.12.6 - #sudo apt-get install -y libltdl7 - #wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb - #sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb - sudo service docker restart + + dce=$(dpkg -l | grep -c docker-ce) + if [[ $dce -eq 0 ]]; then + echo; echo "prereqs.sh: ($(date)) Install latest docker-ce" + # Per https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ + sudo apt-get remove -y docker docker-engine docker.io docker-ce + sudo apt-get update + sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] \ + https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce + fi + echo; echo "prereqs.sh: ($(date)) Get k8s packages" - export KUBE_VERSION=1.7.5 + export KUBE_VERSION=1.10.0 # per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/ # Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/ sudo apt-get update && sudo apt-get install -y apt-transport-https @@ -179,8 +194,15 @@ function setup_k8s_master() { log "Reset kubeadm in case pre-existing cluster" sudo kubeadm reset # Start cluster - log "Start the cluster" + log "Workaround issue '/etc/kubernetes/manifests is not empty'" mkdir ~/tmp + # workaround for [preflight] Some fatal errors occurred: + # /etc/kubernetes/manifests is not empty + sudo rm -rf /etc/kubernetes/manifests/* + log "Disable swap to workaround k8s incompatibility with swap" + # per https://github.com/kubernetes/kubeadm/issues/610 + sudo swapoff -a + log "Start the cluster" sudo kubeadm init --pod-network-cidr=192.168.0.0/16 >>~/tmp/kubeadm.out cat ~/tmp/kubeadm.out export k8s_joincmd=$(grep "kubeadm join" ~/tmp/kubeadm.out) @@ -188,6 +210,7 @@ function setup_k8s_master() { mkdir -p $HOME/.kube sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config + export KUBECONFIG=$HOME/.kube/config # Deploy pod network log "Deploy calico as CNI" # Updated to deploy Calico 2.6 per the create-cluster-kubeadm guide above @@ -198,7 +221,7 @@ function setup_k8s_master() { # Failure to wait for all calico pods to be running can cause the first worker # to be incompletely setup. Symptom is that node_ports cannot be routed # via that node (no response - incoming SYN packets are dropped). - log "Wait for calico pods to be Running" + log "Wait for all calico pods to be Created" # calico-etcd, calico-kube-controllers, calico-node pods=$(kubectl get pods --namespace kube-system | grep -c calico) while [[ $pods -lt 3 ]]; do @@ -207,23 +230,24 @@ function setup_k8s_master() { pods=$(kubectl get pods --namespace kube-system | grep -c calico) done - pods=$(kubectl get pods --all-namespaces | awk '/calico/ {print $2}') + log "Wait for all calico pods to be Running" + pods=$(kubectl get pods --namespace kube-system | awk '/calico/ {print $1}') for pod in $pods; do - status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}") + status=$(kubectl get pods --namespace kube-system | awk "/$pod/ {print \$3}") while [[ "$status" != "Running" ]]; do log "$pod status is $status. Waiting 10 seconds" sleep 10 - status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}") + status=$(kubectl get pods --namespace kube-system | awk "/$pod/ {print \$3}") done log "$pod status is $status" done log "Wait for kubedns to be Running" - kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}') + kubedns=$(kubectl get pods --namespace kube-system | awk '/kube-dns/ {print $3}') while [[ "$kubedns" != "Running" ]]; do log "kube-dns status is $kubedns. Waiting 60 seconds" sleep 60 - kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}') + kubedns=$(kubectl get pods --namespace kube-system | awk '/kube-dns/ {print $3}') done log "kube-dns status is $kubedns" diff --git a/tools/prometheus/prometheus-tools.sh b/tools/prometheus/prometheus-tools.sh index 8463231..05526f6 100644 --- a/tools/prometheus/prometheus-tools.sh +++ b/tools/prometheus/prometheus-tools.sh @@ -29,11 +29,11 @@ #. helm: setup/clean via helm #. agents: for docker-based setup, a quoted, space-separated list agent nodes #. note: node running this script must have ssh-key enabled access to agents -#. $ bash prometheus-tools.sh grafana [URI] [creds] +#. $ bash prometheus-tools.sh grafana [server] [creds] #. grafana: setup/clean grafana #. docker: setup/clean via docker #. helm: setup/clean via helm -#. URI: optional URI of grafana server to use +#. server: optional host:port of grafana server to use #. creds: optional grafana credentials (default: admin:admin) # @@ -60,6 +60,7 @@ function fail() { } function setup_prometheus() { + trap 'fail' ERR log "Setup prometheus" log "Setup prerequisites" if [[ "$dist" == "ubuntu" ]]; then @@ -126,6 +127,7 @@ EOF } function setup_grafana() { + trap 'fail' ERR host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') if [[ "$grafana" == "" ]]; then if [[ "$how" == "docker" ]]; then @@ -138,10 +140,10 @@ function setup_grafana() { log "Setup grafana via Helm" #TODO: add --set server.persistentVolume.storageClass=general helm install --name gf stable/grafana \ - --set server.service.nodePort=30330 \ - --set server.service.type=NodePort \ - --set server.adminPassword=admin \ - --set server.persistentVolume.enabled=false + --set service.nodePort=30330 \ + --set service.type=NodePort \ + --set adminPassword=admin \ + --set persistentVolume.enabled=false fi grafana=$host_ip:30330 fi @@ -264,4 +266,4 @@ case "$1" in ;; *) grep '#. ' $0 -esac \ No newline at end of file +esac -- cgit 1.2.3-korg