From c467bafdce46993f7edcf2db4cce75b6cbe448ae Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Sun, 26 Nov 2017 22:58:15 -0800 Subject: Further cleanup for testing with VES JIRA:MODELS-2 Change-Id: I79914716620ded1e76f15a3c7cd26ad84fa90e10 Signed-off-by: Bryan Sullivan --- .../blueprints/k8s-hello-world/blueprint.yaml | 17 ++++ tools/cloudify/k8s-cloudify.sh | 26 +++--- tools/kubernetes/demo_deploy.sh | 100 ++++++++++++--------- tools/kubernetes/k8s-cluster.sh | 28 ++++-- 4 files changed, 110 insertions(+), 61 deletions(-) diff --git a/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml b/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml index bdbba8c..54c0b45 100644 --- a/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml +++ b/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml @@ -1,4 +1,21 @@ tosca_definitions_version: cloudify_dsl_1_3 +# +# Copyright 2017 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# What this is: Cloudify demo blueprint for running nginx under kubernetes. +# # Based upon # https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/blob/master/examples/simple-blueprint-defined-resource.yaml # https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/blob/1.2.0/examples/example-blueprint.yaml diff --git a/tools/cloudify/k8s-cloudify.sh b/tools/cloudify/k8s-cloudify.sh index 8f52f88..cf6de93 100644 --- a/tools/cloudify/k8s-cloudify.sh +++ b/tools/cloudify/k8s-cloudify.sh @@ -14,18 +14,20 @@ # #. What this is: Setup script for Cloudify use with Kubernetes. #. Prerequisites: -#. - Kubernetes cluster installed per k8s-cluster.sh (in this repo) +#. - OPNFV Models repo cloned into ~/models, i.e. +#. git clone https://gerrit.opnfv.org/gerrit/models ~/models +#. - Kubernetes cluster installed per tools/kubernetes/demo_deploy.sh and +#. environment setup file ~/models/tools/k8s_env.sh as setup by demo_deploy.sh #. Usage: #. From a server with access to the kubernetes master node: -#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models -#. $ cd models/tools/cloudify -#. $ scp -r ~/models/tools/cloudify ubuntu@:/home/ubuntu/. +#. $ cd ~/models/tools/cloudify +#. $ scp -r ~/models/tools/* ubuntu@:/home/ubuntu/. #. : IP or hostname of kubernetes master server #. $ ssh -x ubuntu@ cloudify/k8s-cloudify.sh prereqs #. prereqs: installs prerequisites and configures ubuntu user for kvm use #. $ ssh -x ubuntu@ bash cloudify/k8s-cloudify.sh setup #. setup: installs cloudify CLI and Manager -#. $ bash k8s-cloudify.sh demo +#. $ bash k8s-cloudify.sh demo #. demo: control demo blueprint #. start|stop: start or stop the demo #. : IP or hostname of kubernetes master server @@ -159,7 +161,6 @@ function setup () { function service_port() { name=$1 - manager_ip=$2 tries=6 port="null" while [[ "$port" == "null" && $tries -gt 0 ]]; do @@ -186,7 +187,6 @@ function service_port() { function start() { name=$1 bp=$2 - manager_ip=$3 log "start app $name with blueprint $bp" log "copy kube config from k8s master for insertion into blueprint" scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ @@ -235,7 +235,7 @@ function start() { function stop() { name=$1 bp=$2 - manager_ip=$3 + # TODO: fix the need for this workaround log "try to first cancel all current executions" curl -s -u admin:admin --header 'Tenant: default_tenant' \ @@ -355,7 +355,6 @@ function demo() { # echo "master-port: $(grep server ~/.kube/config | awk -F '/' '{print $3}' | awk -F ':' '{print $2}')" >>~/cloudify/blueprints/k8s-hello-world/inputs.yaml # echo "file_content:" >>~/cloudify/blueprints/k8s-hello-world/inputs.yaml # sed 's/^/ /' ~/.kube/config | tee -a ~/cloudify/blueprints/k8s-hello-world/inputs.yaml - manager_ip=$2 cd ~/models/tools/cloudify/blueprints if [[ "$1" == "start" ]]; then @@ -377,6 +376,9 @@ function clean () { } dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'` +source ~/k8s_env.sh +manager_ip=$k8s_master + case "$1" in "prereqs") prereqs @@ -388,13 +390,13 @@ case "$1" in demo $2 $3 ;; "start") - start $2 $3 $4 + start $2 $3 ;; "stop") - stop $2 $3 $4 + stop $2 $3 ;; "port") - service_port $2 $3 + service_port $2 ;; "clean") clean diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh index 4777c4b..0c581b8 100644 --- a/tools/kubernetes/demo_deploy.sh +++ b/tools/kubernetes/demo_deploy.sh @@ -39,37 +39,57 @@ #. : disk (e.g. sda, sdb) or folder (e.g. "/ceph") #. : optional name of script for extra setup functions as needed -key=$1 -nodes="$2" -master=$3 -workers="$4" -priv_net=$5 -pub_net=$6 -ceph_mode=$7 -ceph_dev=$8 -extras=$9 - function run_master() { -ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$master <~/k8s_env.sh +k8s_key=$1 +k8s_nodes="$2" +k8s_master=$3 +k8s_workers="$4" +k8s_priv_net=$5 +k8s_pub_net=$6 +k8s_ceph_mode=$7 +k8s_ceph_dev=$8 +export k8s_key +export k8s_nodes +export k8s_master +export k8s_workers +export k8s_priv_net +export k8s_pub_net +export k8s_ceph_mode +export k8s_ceph_dev +EOF +source ~/k8s_env.sh +env | grep k8s_ + +source ~/models/tools/maas/deploy.sh $k8s_key "$k8s_nodes" $extras eval `ssh-agent` -ssh-add $key -scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $key ubuntu@$master:/home/ubuntu/$key +ssh-add $k8s_key +scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_key \ + ubuntu@$k8s_master:/home/ubuntu/$k8s_key +scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ~/k8s_env.sh \ + ubuntu@$k8s_master:/home/ubuntu/. echo; echo "$0 $(date): Setting up kubernetes master..." -scp -r -o StrictHostKeyChecking=no ~/models/tools/kubernetes/* \ - ubuntu@$master:/home/ubuntu/. +scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ~/models/tools/kubernetes/* ubuntu@$k8s_master:/home/ubuntu/. run_master "bash k8s-cluster.sh master" echo; echo "$0 $(date): Setting up kubernetes workers..." -run_master "bash k8s-cluster.sh workers \"$workers\"" +run_master "bash k8s-cluster.sh workers \"$k8s_workers\"" echo; echo "$0 $(date): Setting up helm..." run_master "bash k8s-cluster.sh helm" @@ -79,49 +99,49 @@ run_master "bash k8s-cluster.sh demo start nginx" run_master "bash k8s-cluster.sh demo stop nginx" echo; echo "$0 $(date): Setting up ceph-helm" -run_master "bash k8s-cluster.sh ceph \"$workers\" $priv_net $pub_net $ceph_mode $ceph_dev" +run_master "bash k8s-cluster.sh ceph \"$k8s_workers\" $k8s_priv_net $k8s_pub_net $k8s_ceph_mode $k8s_ceph_dev" echo; echo "$0 $(date): Verifying kubernetes+helm+ceph install..." run_master "bash k8s-cluster.sh demo start dokuwiki" echo; echo "Setting up Prometheus..." scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \ - ubuntu@$master:/home/ubuntu/. -run_master "bash prometheus-tools.sh all \"$workers\"" + ubuntu@$k8s_master:/home/ubuntu/. +run_master "bash prometheus-tools.sh all \"$k8s_workers\"" echo; echo "$0 $(date): Setting up cloudify..." scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \ - ubuntu@$master:/home/ubuntu/. + ubuntu@$k8s_master:/home/ubuntu/. run_master "bash cloudify/k8s-cloudify.sh prereqs" run_master "bash cloudify/k8s-cloudify.sh setup" echo; echo "$0 $(date): Verifying kubernetes+helm+ceph+cloudify install..." -bash ~/models/tools/cloudify/k8s-cloudify.sh demo start $master +bash ~/models/tools/cloudify/k8s-cloudify.sh demo start -echo; echo "$0 $(date): Setting up VES master node" +echo; echo "$0 $(date): Setting up VES" +# not re-cloned if existing - allows patch testing locally if [[ ! -d ~/ves ]]; then git clone https://gerrit.opnfv.org/gerrit/ves ~/ves fi -ves_grafana_host=$master:3000 -ves_grafana_auth=admin:admin +ves_influxdb_host=$k8s_master:8086 +export ves_influxdb_host +ves_grafana_host=$k8s_master:3000 export ves_grafana_host +ves_grafana_auth=admin:admin export ves_grafana_auth -bash ~/ves/tools/demo_deploy.sh master $master $key - -echo; echo "$0 $(date): Setting up collectd for VES events from worker nodes" -for worker in $workers; do -bash ~/ves/tools/demo_deploy.sh worker $worker $key -done +ves_kafka_hostname=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master hostname) +export ves_kafka_hostname +bash ~/ves/tools/demo_deploy.sh $k8s_key $k8s_master "$k8s_workers" cloudify echo; echo "$0 $(date): All done!" -export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki) -export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") +export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki) +export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/" # TODO update Cloudify demo app to have public exposed service address -port=$( bash ~/models/tools/cloudify/k8s-cloudify.sh port nginx $master) -echo "Cloudify-deployed demo app nginx is available at http://$master:$port" -echo "Prometheus UI is available at http://$master:9090" +port=$( bash ~/models/tools/cloudify/k8s-cloudify.sh port nginx $k8s_master) +echo "Cloudify-deployed demo app nginx is available at http://$k8s_master:$port" +echo "Prometheus UI is available at http://$k8s_master:9090" echo "Grafana dashboards are available at http://$ves_grafana_host (login as $ves_grafana_auth)" echo "Grafana API is available at http://$ves_grafana_auth@$ves_influx_host/api/v1/query?query=" -echo "Kubernetes API is available at https://$master:6443/api/v1/" -echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$master/api/v3.1/status" +echo "Kubernetes API is available at https://$k8s_master:6443/api/v1/" +echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$k8s_master/api/v3.1/status" diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index 0d351ba..edbe93c 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -66,7 +66,8 @@ sudo apt-get update sudo apt-get upgrade -y if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then echo; echo "prereqs.sh: ($(date)) Add $HOSTNAME to /etc/hosts" - echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" | sudo tee -a /etc/hosts + echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" \ + | sudo tee -a /etc/hosts fi echo; echo "prereqs.sh: ($(date)) Install latest docker" sudo apt-get install -y docker.io @@ -86,7 +87,8 @@ deb http://apt.kubernetes.io/ kubernetes-xenial main EOF sudo apt-get update echo; echo "prereqs.sh: ($(date)) Install kubectl, kubelet, kubeadm" -sudo apt-get -y install --allow-downgrades kubectl=${KUBE_VERSION}-00 kubelet=${KUBE_VERSION}-00 kubeadm=${KUBE_VERSION}-00 +sudo apt-get -y install --allow-downgrades kubectl=${KUBE_VERSION}-00 \ + kubelet=${KUBE_VERSION}-00 kubeadm=${KUBE_VERSION}-00 echo; echo "prereqs.sh: ($(date)) Install jq for API output parsing" sudo apt-get -y install jq echo; echo "prereqs.sh: ($(date)) Set firewall rules" @@ -121,7 +123,8 @@ function setup_k8s_master() { bash /tmp/prereqs.sh master # per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/ # If the following command fails, run "kubeadm reset" before trying again - # --pod-network-cidr=192.168.0.0/16 is required for calico; this should not conflict with your server network interface subnets + # --pod-network-cidr=192.168.0.0/16 is required for calico; this should not + # conflict with your server network interface subnets sudo kubeadm init --pod-network-cidr=192.168.0.0/16 >>/tmp/kubeadm.out cat /tmp/kubeadm.out export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out) @@ -156,16 +159,23 @@ function setup_k8s_workers() { for worker in $workers; do log "Install worker at $worker" - if ! scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/prereqs.sh ubuntu@$worker:/tmp/prereqs.sh ; then + if ! scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + /tmp/prereqs.sh ubuntu@$worker:/tmp/prereqs.sh ; then fail "Failed copying setup files to $worker" fi - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker bash /tmp/prereqs.sh worker - # Workaround for "[preflight] Some fatal errors occurred: /var/lib/kubelet is not empty" per https://github.com/kubernetes/kubeadm/issues/1 - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker sudo kubeadm reset - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker sudo $k8s_joincmd + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ~/k8s_env.sh \ + ubuntu@$worker:/home/ubuntu/. + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$worker bash /tmp/prereqs.sh worker + # Workaround for "[preflight] Some fatal errors occurred: /var/lib/kubelet + # is not empty" per https://github.com/kubernetes/kubeadm/issues/1 + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$worker sudo kubeadm reset + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$worker sudo $k8s_joincmd done - log "Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'." + log "Cluster is ready when all nodes in 'kubectl get nodes' show as 'Ready'." } function setup_ceph() { -- cgit 1.2.3-korg