Split setup and clean scripts. Fix daemonset for barometer.

JIRA: VES-2

Change-Id: Ied6f84500284b75976df2acb6d341e30fff35e6e
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

4 files changed, 288 insertions, 102 deletions

diff --git a/tools/demo_deploy.sh b/tools/demo_deploy.sh
index 1105c2a..5567ce6 100644
--- a/tools/demo_deploy.sh
+++ b/tools/demo_deploy.sh
@@ -115,13 +115,13 @@ function deploy() {
   ves_grafana_auth=$ves_grafana_auth
   ves_loglevel=$ves_loglevel
   source ~/ves/tools/ves-setup.sh env
-  env | grep ves_ >~/ves/tools/ves_env.sh
-  for var in $vars; do echo "export $var" | tee -a ~/ves/tools/ves_env.sh; done
 
   log "Setting up master node"
   run_master "mkdir /home/$user/ves"
   scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
     ~/ves/tools $user@$master:/home/$user/ves
+  run "bash ves/tools/ves-setup.sh influxdb"
+  run "bash ves/tools/ves-setup.sh grafana"
   run "bash ves/tools/ves-setup.sh collector $cloudify"
   run "bash ves/tools/ves-setup.sh kafka $cloudify"
   run "bash ves/tools/ves-setup.sh agent $cloudify"
@@ -133,14 +133,14 @@ function deploy() {
   fi
 
   for node in $nodes; do
-    log "Setting up collectd at $node"
+    log "Setting up barometer at $node"
     if [[ "$node" != "$k8s_master" ]]; then
       ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
         $user@$node mkdir /home/$user/ves
       scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
        ~/ves/tools $user@$node:/home/$user/ves
     fi
-    run "bash ves/tools/ves-setup.sh collectd"
+    run "bash ves/tools/ves-setup.sh barometer"
 EOF
   done
 
@@ -152,7 +152,7 @@ deploy_start=$((`date +%s`/60))
 user=$1
 master=$2
 cloudify=$3
-source ~/k8s_env_$master.sh
+source ~/k8s_env_$k8s_master_hostname.sh
 log "k8s environment as input"
 env | grep k8s
 eval `ssh-agent`
diff --git a/tools/kubernetes/ves-barometer/daemonset.yaml b/tools/kubernetes/ves-barometer/daemonset.yaml
index d0d6d19..8f4c1d9 100644
--- a/tools/kubernetes/ves-barometer/daemonset.yaml
+++ b/tools/kubernetes/ves-barometer/daemonset.yaml
@@ -13,9 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-#  What this is: Kuberbetes chart for the OPNFV Barometer collectd agent
-#  running as a daemonset (one each kluster node) under kubernetes.
-apiVersion: apps/v1
+# What this is: Kuberbetes chart for the OPNFV Barometer collectd agent
+# running as a daemonset (one each kluster node) under kubernetes.
+#
+# Notes:
+# - apiVersion: extensions/v1beta1 is required for Ubuntu (apps/v1 failed with
+#   "error: unable to recognize... no matches for apps/, Kind=DaemonSet"
+apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
   name: ves-barometer
diff --git a/tools/ves-clean.sh b/tools/ves-clean.sh
new file mode 100644
index 0000000..0f9b8d7
--- /dev/null
+++ b/tools/ves-clean.sh
@@ -0,0 +1,230 @@
+#!/bin/bash
+# Copyright 2017-2018 AT&T Intellectual Property, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: Cleanup script for the VES monitoring framework.
+#. With this script a VES deployment can be cleaned from one or more hosts.
+#.
+#. Prerequisites:
+#. - VES framework deployed as in ves-setup.sh in this repo
+#.
+#. Usage:
+#.   bash ~/ves/ves-setup.sh clean <what> [cloudify]
+#.     what: one of all|influxdb|grafana|collector|kafka|collectd|agent|nodes
+#.     barometer: clean barometer
+#.     agent: clean VES agent
+#.     kafka: clean kafka
+#.     zookeeper: clean zookeeper
+#.     grafana: clean grafana 
+#.     influxdb: clean influxdb 
+#.     collector: clean VES collector
+#.     nodes: clean VES code etc at nodes
+#.     cloudify: (optional) clean up cloudify-based deployments
+#.
+#.   See demo_deploy.sh in this repo for a recommended sequence of the above.
+#.
+#. Status: this is a work in progress, under test.
+
+# http://docs.opnfv.org/en/latest/submodules/barometer/docs/release/userguide/collectd.ves.userguide.html
+
+function fail() {
+  log "$1"
+  exit 1
+}
+
+function log() {
+  f=$(caller 0 | awk '{print $2}')
+  l=$(caller 0 | awk '{print $1}')
+  echo ""
+  echo "$f:$l ($(date)) $1"
+}
+
+function clean_all() {
+  log "clean installation"
+  clean_barometer
+  clean_agent
+  clean_kafka
+  clean_zookeeper
+  clean_grafana
+  clean_influxdb
+  clean_collector
+  clean_nodes  
+}
+
+function clean_via_docker() {
+  log "clean docker container $1 at k8s master $k8s_master"
+  ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+    $k8s_user@$k8s_master <<EOF
+sudo docker stop $1
+sudo docker rm -v $1
+EOF
+}
+
+function clean_via_cloudify() {
+  log "clean $1 via cloudify"
+  bash ~/models/tools/cloudify/k8s-cloudify.sh stop $1 $1
+}
+
+
+function clean_grafana() {
+  log "clean grafana"
+
+  log "VES datasources and dashboards at grafana server, if needed"
+  curl -X DELETE \
+    http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/datasources/name/VESEvents
+  curl -X DELETE \
+    http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/dashboards/db/ves-demo
+
+  clean_via_docker ves-grafana
+}
+
+function clean_influxdb() {
+  log "clean influxdb"
+  clean_via_docker ves-influxdb
+}
+
+function clean_agent() {
+  log "clean ves-agent"
+  if [[ "$cloudify" == "cloudify" ]]; then
+    clean_via_cloudify ves-agent
+    force_k8s_clean ves-agent
+  else
+    clean_via_docker ves-agent
+  fi
+}
+
+function clean_kafka() {
+  log "clean ves-kafka"
+  if [[ "$cloudify" == "cloudify" ]]; then
+    clean_via_cloudify ves-kafka
+    force_k8s_clean ves-kafka
+  else
+    clean_via_docker ves-kafka
+  fi
+}
+
+function clean_zookeeper() {
+  log "clean ves-zookeeper"
+  if [[ "$cloudify" == "cloudify" ]]; then
+    clean_via_cloudify ves-zookeeper
+    force_k8s_clean ves-zookeeper
+  else
+    clean_via_docker ves-zookeeper
+  fi
+}
+
+function clean_collector() {
+  log "clean ves-zookeeper"
+  if [[ "$cloudify" == "cloudify" ]]; then
+    clean_via_cloudify ves-collector
+    force_k8s_clean ves-collector
+  else
+    clean_via_docker ves-collector
+  fi
+}
+
+function clean_barometer() {
+  log "clean ves-barometer"
+  ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+kubectl delete daemonset --namespace default ves-barometer
+EOF
+  force_k8s_clean ves-barometer
+}
+
+function clean_nodes() {
+  log "clean ves code etc from nodes"
+  if [[ "$k8s_master" == "$k8s_workers" ]]; then
+    nodes=$k8s_master
+  else
+    nodes="$k8s_master $k8s_workers"
+  fi
+  for node in $nodes; do 
+    log "remove ves-barometer container and config for VES at node $node"
+    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+      $k8s_user@$node <<EOF
+sudo rm -rf /home/$k8s_user/ves
+sudo rm -rf /home/$k8s_user/collectd
+EOF
+  done
+}
+
+function force_k8s_clean() {
+  log "force cleanup of k8s pod for $1 if still present"
+  ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+    $k8s_user@$k8s_master "kubectl delete pods --namespace default $1-pod"
+  pods=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get pods --namespace default | grep -c $1)
+  echo "wait for all kubectl pods to be terminated"
+  tries=10
+  while [[ $pods -gt 0 && $tries -gt 0 ]]; do
+    echo "$pods VES pods remaining in kubectl"
+    sleep 30
+    ((tries--))
+    pods=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get pods --namespace default | grep -c $1)
+  done
+  if [[ $pods -gt 0 ]]; then
+    log "manually terminate $1 pods via docker"
+    cs=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master sudo docker ps -a | awk "/$1/ {print $1}") 
+    for c in $cs ; do
+      ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master "sudo docker stop $c"
+      ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master "sudo docker rm -v $c"
+    done
+  fi
+}
+
+dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g')
+if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then 
+  echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" |\
+    sudo tee -a /etc/hosts
+fi
+
+source ~/k8s_env_$k8s_master_hostname.sh
+if [[ -f ~/ves/tools/ves_env.sh ]]; then
+  source ~/ves/tools/ves_env.sh
+fi
+log "VES environment as input"
+env | grep ves_
+
+trap 'fail' ERR
+
+cloudify=$2
+
+case "$1" in
+  "all")
+    clean_all
+    ;;
+  "modes")
+    clean_nodes
+    ;;
+  "collectd")
+    clean_collectd
+    ;;
+  "agent")
+    clean_agent
+    ;;
+  "influxdb")
+    clean_influxdb
+    ;;
+  "grafana")
+    clean_grafana
+    ;;
+  "collector")
+    clean_collector
+    ;;
+  "kafka")
+    clean_kafka
+    ;;
+  *)
+    grep '#. ' $0
+esac
+trap '' ERR
diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh
index 1a3dbbe..aade964 100644
--- a/tools/ves-setup.sh
+++ b/tools/ves-setup.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2017 AT&T Intellectual Property, Inc
+# Copyright 2017-2018 AT&T Intellectual Property, Inc
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,8 +15,8 @@
 #
 #. What this is: Setup script for the VES monitoring framework.
 #. With this script VES support can be installed in one or more hosts, with:
-#. - a dedicated or shared Kafka server for collection of events from collectd
-#. - VES collectd agents running in host or guest mode
+#. - a dedicated or shared Kafka server for collection of events from barometer
+#. - VES barometer agents running in host or guest mode
 #. - VES monitor (test collector)
 #. - Influxdb service (if an existing service is not passed as an option)
 #. - Grafana service (if an existing service is not passed as an option)
@@ -26,14 +26,14 @@
 #.    pre-installed VES collector e.g. from the ONAP project.
 #.  - Install Kafka server on one of the hosts, or use a pre-installed server
 #.    accessible from the agent hosts.
-#.  - Install collectd on each host.
+#.  - Install barometer on each host.
 #.  - Install the VES agent on one of the hosts.
 #.
 #. Prerequisites:
 #. - Ubuntu Xenial (Centos support to be provided)
 #. - passwordless sudo setup for user running this script
 #. - shell environment variables setup as below (for non-default setting)
-#.   ves_mode: install mode (node|guest) for VES collectd plugin (default: node)
+#.   ves_mode: install mode (node|guest) for VES barometer plugin (default: node)
 #.   ves_host: VES collector IP or hostname (default: 127.0.0.1)
 #.   ves_port: VES collector port (default: 3001)
 #.   ves_path: REST path optionalRoutingPath element (default: empty)
@@ -41,7 +41,7 @@
 #.   ves_https: use HTTPS instead of HTTP (default: false)
 #.   ves_user: username for basic auth with collector (default: empty)
 #.   ves_pass: password for basic auth with collector (default: empty)
-#.   ves_interval: frequency in sec for collectd data reports (default: 20)
+#.   ves_interval: frequency in sec for barometer data reports (default: 20)
 #.   ves_version: VES API version (default: 5.1)
 #.   ves_kafka_host: kafka host IP (default: 127.0.0.1)
 #.   ves_kafka_hostname: kafka host hostname (default: localhost)
@@ -54,20 +54,28 @@
 #.
 #. Usage:
 #.   git clone https://gerrit.opnfv.org/gerrit/ves ~/ves
-#.   bash ~/ves/tools/ves-setup.sh <env|collector|kafka|collectd|agent> [cloudify]
+#.   bash ~/ves/tools/ves-setup.sh <what> [cloudify]
+#.     what: one of env|influxdb|grafana|collector|zookeeper|kafka|agent|barometer
 #.     env: setup VES environment script ~/ves/tools/ves_env.sh 
+#.     influxdb: setup influxdb as a docker container on k8s_master node 
+#.     grafana: setup grafana as a docker container on k8s_master node 
 #.     collector: setup VES collector (test collector) 
+#.     zookeeper: setup zookeeper server for kafka configuration
 #.     kafka: setup kafka server for VES events from collect agent(s)
-#.     collectd: setup collectd with libvirt plugin, as a kafka publisher
 #.     agent: setup VES agent in host or guest mode, as a kafka consumer
+#.     barometer: setup barometer with libvirt plugin, as a kafka publisher
 #.     cloudify: (optional) use cloudify to deploy the component, as setup by
 #.       tools/cloudify/k8s-cloudify.sh in the OPNFV Models repo.
-#.   bash ~/ves/ves-setup.sh clean <master> <workers> [cloudify]
-#.     master: VES master node IP
-#.     workers: quoted, space-separated list of worker node IPs
-#.     cloudify: (optional) clean up cloudify-based deployments
 #.
-#.   See demo_deploy.sh in this repo for a recommended sequence of the above.
+#.   The recommended sequence for setting up the components is:
+#.     influxdb: prerequisite for grafana datasource setup
+#.     grafana:  prerequisite for setup of datasource and dashboards
+#.     collector: creates veseventsdb in influxdb, and grafana 
+#.       datasource/dashboards, then starts listening for VES event reports
+#.     zookeeper: prerequisite for kafka
+#.     kafka: prerequisite for agent and barometer
+#.     agent: listens for collectd topic events over kafka, for reporting to collector
+#.     barometer: monitors resources and reports via collectd topic in kafka
 #.
 #. Status: this is a work in progress, under test.
 
@@ -101,7 +109,7 @@ function setup_env() {
   cat <<EOF >~/ves/tools/ves_env.sh
 #!/bin/bash
 ves_mode="${ves_mode:=node}"
-ves_host="${ves_host:=127.0.0.1}"
+ves_host="${ves_host:=ves-collector-service.default.svc.cluster.local}"
 ves_hostname="${ves_hostname:=ves-collector-service.default.svc.cluster.local}"
 ves_port="${ves_port:=3001}"
 ves_path="${ves_path:=}"
@@ -111,17 +119,18 @@ ves_user="${ves_user:=}"
 ves_pass="${ves_pass:=}"
 ves_interval="${ves_interval:=20}"
 ves_version="${ves_version:=5.1}"
+ves_zookeeper_host="${ves_zookeeper_host:=ves-zookeeper-service.default.svc.cluster.local}"
 ves_zookeeper_hostname="${ves_zookeeper_hostname:=ves-zookeeper-service.default.svc.cluster.local}"
-ves_zookeeper_host="${ves_zookeeper_host:=127.0.0.1}"
+ves_zookeeper_host="${ves_zookeeper_host:=ves-zookeeper-service.default.svc.cluster.local}"
 ves_zookeeper_port="${ves_zookeeper_port:=2181}"
-ves_kafka_host="${ves_kafka_host:=127.0.0.1}"
+ves_kafka_host="${ves_kafka_host:=ves-kafka-service.default.svc.cluster.local}"
 ves_kafka_hostname="${ves_kafka_hostname:=ves-kafka-service.default.svc.cluster.local}"
 ves_kafka_port="${ves_kafka_port:=9092}"
-ves_influxdb_host="${ves_influxdb_host:=127.0.0.1}"
+ves_influxdb_host="${ves_influxdb_host:=ves-influxdb-service.default.svc.cluster.local}"
 ves_influxdb_hostname="${ves_influxdb_hostname:=ves-influxdb-service.default.svc.cluster.local}"
 ves_influxdb_port="${ves_influxdb_port:=8086}"
 ves_influxdb_auth="${ves_influxdb_auth:=}"
-ves_grafana_host="${ves_grafana_host:=127.0.0.1}"
+ves_grafana_host="${ves_grafana_host:=ves-grafana-service.default.svc.cluster.local}"
 ves_grafana_hostname="${ves_grafana_hostname:=ves-grafana-service.default.svc.cluster.local}"
 ves_grafana_port="${ves_grafana_port:=3000}"
 ves_grafana_auth="${ves_grafana_auth:=admin:admin}"
@@ -204,8 +213,8 @@ EOF
   fi
 }
 
-function setup_collectd() {
-  log "setup collectd"
+function setup_barometer() {
+  log "setup barometer"
 #  if [[ $(grep -c $ves_kafka_hostname /etc/hosts) -eq 0 ]]; then
 #    log "add to /etc/hosts: $ves_kafka_host $ves_kafka_hostname"
 #    echo "$ves_kafka_host $ves_kafka_hostname" | sudo tee -a /etc/hosts
@@ -284,9 +293,8 @@ EOF
   # ~/kafka_2.11-0.11.0.2/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic collectd
 }
 
-function setup_collector() {
-  log "setup collector"
-
+function setup_influxdb() {
+  log "setup influxdb"
   log "install prerequistes"
   if [[ "$dist" == "ubuntu" ]]; then
     sudo apt-get install -y jq
@@ -313,7 +321,10 @@ done
 echo; echo "InfluxDB container state is $status"
 EOF
   fi
+}
 
+function setup_grafana() {
+  log "setup grafana"
   log "checking for grafana at http://$ves_grafana_host:$ves_grafana_port"
   if ! curl http://$ves_grafana_host:$ves_grafana_port ; then
     log "install Grafana container on k8s master"
@@ -333,8 +344,10 @@ done
 echo; echo "Grafana container state is $status"
 EOF
   fi
+}
 
-  log "setup collector container"
+function setup_collector() {
+  log "setup collector"
   if [[ "$1" == "cloudify" ]]; then
     cp -r ~/ves/tools/cloudify/ves-collector ~/models/tools/cloudify/blueprints/.
     inputs="{ \
@@ -390,70 +403,6 @@ EOF
   # sudo docker exec -it ves-collector /bin/bash
 }
 
-function clean() {
-  log "clean installation"
-  master=$1
-  workers="$2"
-  cloudify=$3
-
-  log "VES datasources and dashboards at grafana server, if needed"
-  curl -X DELETE \
-    http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/datasources/name/VESEvents
-  curl -X DELETE \
-    http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/dashboards/db/ves-demo
-
-  if [[ "$cloudify" == "cloudify" ]]; then
-    log "stop cloudify-managed services"
-    bash ~/models/tools/cloudify/k8s-cloudify.sh stop ves-agent ves-agent
-    bash ~/models/tools/cloudify/k8s-cloudify.sh stop ves-kafka ves-kafka
-    bash ~/models/tools/cloudify/k8s-cloudify.sh stop ves-zookeeper ves-zookeeper
-    bash ~/models/tools/cloudify/k8s-cloudify.sh stop ves-collector ves-collector
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-      $k8s_user@$k8s_master <<'EOF'
-cs="ves-grafana ves-influxdb"
-for c in $cs; do
-  sudo docker stop $c
-  sudo docker rm -v $c
-done
-kubectl delete daemonset --namespace default ves-barometer
-EOF
-    log "workaround: force cleanup of k8s pods if still present"
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-      $k8s_user@$k8s_master <<'EOF'
-cs="ves-agent-pod ves-collector-pod ves-grafana-pod ves-influxdb-pod ves-kafka-pod ves-zookeeper-pod"
-for c in $cs; do
-  kubectl delete pods --namespace default $c
-done
-EOF
-  else
-    log "Remove VES containers and collectd config at master node"
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-      $k8s_user@$k8s_master <<'EOF'
-cs="ves-agent ves-collector ves-grafana ves-influxdb ves-kafka ves-zookeeper"
-for c in $cs; do
-  sudo docker stop $c
-  sudo docker rm -v $c
-done
-EOF
-  fi
-
-  if [[ "$master" == "$workers" ]]; then
-    nodes=$master
-  else
-    nodes="$master $workers"
-  fi
-  for node in $nodes; do 
-    log "remove ves-barometer container and config for VES at node $node"
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-      $k8s_user@$node <<EOF
-sudo docker stop ves-barometer
-sudo docker rm -v ves-barometer
-sudo rm -rf /home/$k8s_user/ves
-sudo rm -rf /home/$k8s_user/collectd
-EOF
-  done
-}
-
 function verify_veseventsdb() {
   log "VES environment as set by ves_env.sh"
   env | grep ves
@@ -476,7 +425,7 @@ if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then
     sudo tee -a /etc/hosts
 fi
 
-source ~/k8s_env.sh
+source ~/k8s_env_$k8s_master_hostname.sh
 if [[ -f ~/ves/tools/ves_env.sh ]]; then
   source ~/ves/tools/ves_env.sh
 fi
@@ -489,12 +438,18 @@ case "$1" in
   "env")
     setup_env
     ;;
-  "collectd")
-    setup_collectd
+  "barometer")
+    setup_barometer
     ;;
   "agent")
     setup_agent $2
     ;;
+  "influxdb")
+    setup_influxdb
+    ;;
+  "grafana")
+    setup_grafana
+    ;;
   "collector")
     setup_collector $2
     ;;
@@ -504,9 +459,6 @@ case "$1" in
   "verify")
     verify_veseventsdb "$1" load load-shortterm
     ;;
-  "clean")
-    clean $2 "$3" $4
-    ;;
   *)
     grep '#. ' $0
 esac