Updated monitor.py and grafana dashboard

JIRA: VES-2
Various fixes in ves-setup
Add demo_deploy.sh
Refactored to containerized agent and collector

Change-Id: I8851465742aaf40a4cce265508a3d2d66abced08
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

1 files changed, 157 insertions, 122 deletions

diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh
index 65dd613..b37662b 100644
--- a/tools/ves-setup.sh
+++ b/tools/ves-setup.sh
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-#. What this is: Setup script for VES agent framework.
+#. What this is: Setup script for the VES monitoring framework.
 #. With this script VES support can be installed in one or more hosts, with:
 #. - a dedicated or shared Kafka server for collection of events from collectd
 #. - VES collectd agents running in host or guest mode
@@ -23,6 +23,7 @@
 #.    pre-installed VES collector e.g. from the ONAP project.
 #.  - Install Kafka server on one of the hosts, or use a pre-installed server
 #.    accessible from the agent hosts.
+#.  - Install collectd on each host.
 #.  - Install the VES agent on each host.
 #.  - As needed, install the VES agent on each virtual host. This could include
 #.    pre-installed VES agents in VM or container images, which are configured
@@ -47,21 +48,13 @@
 #.   ves_kafka_host: kafka host IP or hostname (default: 127.0.0.1)
 #.
 #. Usage:
-#.   wget https://raw.githubusercontent.com/opnfv/ves/master/tools/ves-setup.sh
-#.   bash ves-setup.sh <collector|kafka|agent>
+#.   git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves
+#.   bash /tmp/ves/ves-setup.sh <collector|kafka|collectd|agent>
 #.     collector: setup VES collector (test collector) 
 #.     kafka: setup kafka server for VES events from collect agent(s)
-#.     agent: setup VES agent in host or guest mode
-#.   Recommended sequence is:
-#.     ssh into your collector host and run these commands:
-#.     $ ves_host=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') 
-#.     $ export ves_host
-#.     $ bash ves-setup.sh collector 
-#.   ...then for each agent host:
-#.     copy ~/ves_env.sh and ves-setup.sh to the host e.g. via scp
-#.     ssh into the host and run, directly or via ssh -x
-#.     $ bash ves-setup.sh kafka
-#.     $ bash ves-setup.sh agent
+#.     collectd: setup collectd with libvirt plugin, as a kafka publisher
+#.     agent: setup VES agent in host or guest mode, as a kafka consumer
+#.   See demo_deploy.sh in this repo for a recommended sequence of the above.
 #.
 #. Status: this is a work in progress, under test.
 
@@ -81,18 +74,27 @@ function log() {
 
 function common_prereqs() {
   log "install common prerequisites"
-  sudo apt-get install -y default-jre
-  sudo apt-get install -y zookeeperd
-  sudo apt-get install -y python-pip
-  sudo pip install kafka-python
+  if [[ ! -f /.dockerenv ]]; then dosudo="sudo"; fi
+  $dosudo apt-get update
+  $dosudo apt-get install -y git
+  # Required for kafka
+  $dosudo apt-get install -y default-jre
+  $dosudo apt-get install -y zookeeperd
+  $dosudo apt-get install -y python-pip
+  $dosudo pip install kafka-python
+  # Required for building collectd
+  $dosudo apt-get install -y pkg-config
 }
 
 function setup_env() {
-  if [[ ! -f ~/ves_env.sh ]]; then
-    cat <<EOF >~/ves_env.sh
+  if [[ ! -d /tmp/ves ]]; then mkdir /tmp/ves; fi
+  cp $0 /tmp/ves
+  if [[ ! -f /tmp/ves/ves_env.sh ]]; then
+    cat <<EOF >/tmp/ves/ves_env.sh
 #!/bin/bash
 ves_mode="${ves_mode:=host}"
 ves_host="${ves_host:=127.0.0.1}"
+ves_hostname="${ves_hostname:=localhost}"
 ves_port="${ves_port:=30000}"
 ves_path="${ves_path:=}"
 ves_topic="${ves_topic:=}"
@@ -105,6 +107,7 @@ ves_kafka_host="${ves_kafka_host:=127.0.0.1}"
 ves_kafka_port="${ves_kafka_port:=9092}"
 export ves_mode
 export ves_host
+export ves_hostname
 export ves_port
 export ves_path
 export ves_topic
@@ -112,57 +115,62 @@ export ves_https
 export ves_user
 export ves_pass
 export ves_interval
-export ves_kafka_host
-export ves_port
+export ves_kafka_hostame
 export ves_kafka_port
 EOF
   fi
-  source ~/ves_env.sh
+  source /tmp/ves/ves_env.sh
 }
 
 function setup_kafka() {
   log "setup kafka server"
   common_prereqs
+  setup_env
 
-  log "get and unpack kafka_2.11-0.11.0.0.tgz"
-  wget "http://www-eu.apache.org/dist/kafka/0.11.0.0/kafka_2.11-0.11.0.0.tgz"
-  tar -xvzf kafka_2.11-0.11.0.0.tgz
+  cd
+  ver="0.11.0.2"
+  log "get and unpack kafka_2.11-$ver.tgz"
+  wget "http://www-eu.apache.org/dist/kafka/$ver/kafka_2.11-$ver.tgz"
+  tar -xvzf kafka_2.11-$ver.tgz
 
   log "set delete.topic.enable=true"
   sed -i -- 's/#delete.topic.enable=true/delete.topic.enable=true/' \
-    kafka_2.11-0.11.0.0/config/server.properties
-  grep delete.topic.enable kafka_2.11-0.11.0.0/config/server.properties
+    kafka_2.11-$ver/config/server.properties
+  grep delete.topic.enable kafka_2.11-$ver/config/server.properties
   # TODO: Barometer VES guide to clarify hostname must be in /etc/hosts
-  sudo nohup kafka_2.11-0.11.0.0/bin/kafka-server-start.sh \
-    kafka_2.11-0.11.0.0/config/server.properties \
-    > kafka_2.11-0.11.0.0/kafka.log 2>&1 &
-  # TODO: find a test that does not hang the script at 
-  # echo "Hello, World" | ~/kafka_2.11-0.11.0.0/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic TopicTest > /dev/null
-  # ~/kafka_2.11-0.11.0.0/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic TopicTest --from-beginning
+  sudo nohup kafka_2.11-$ver/bin/kafka-server-start.sh \
+    kafka_2.11-$ver/config/server.properties \
+    > kafka_2.11-$ver/kafka.log 2>&1 &
 }
 
-function setup_agent() {
-  log "setup agent"
-  common_prereqs
-
-  log "cleanup any previous failed install"
-  sudo rm -rf ~/collectd-virt
-  sudo rm -rf ~/librdkafka
-  sudo rm -rf ~/collectd
-
+function setup_kafka_client() {
   log "Install Apache Kafka C/C++ client library"
-  sudo apt-get install -y build-essential
+  if [[ ! -f /.dockerenv ]]; then dosudo="sudo"; fi
+  $dosudo apt-get install -y build-essential
   git clone https://github.com/edenhill/librdkafka.git ~/librdkafka
   cd ~/librdkafka
   git checkout -b v0.9.5 v0.9.5
   # TODO: Barometer VES guide to clarify specific prerequisites for Ubuntu
-  sudo apt-get install -y libpthread-stubs0-dev
-  sudo apt-get install -y libssl-dev
-  sudo apt-get install -y libsasl2-dev
-  sudo apt-get install -y liblz4-dev
+  $dosudo apt-get install -y libpthread-stubs0-dev
+  $dosudo apt-get install -y libssl-dev
+  $dosudo apt-get install -y libsasl2-dev
+  $dosudo apt-get install -y liblz4-dev
   ./configure --prefix=/usr
   make
-  sudo make install
+  $dosudo make install
+}
+
+function setup_collectd() {
+  log "setup collectd"
+  common_prereqs
+  setup_env
+
+  log "cleanup any previous failed install"
+  sudo rm -rf ~/collectd-virt
+  sudo rm -rf ~/librdkafka
+  sudo rm -rf ~/collectd
+
+  setup_kafka_client
 
   log "Build collectd with Kafka support"
   git clone https://github.com/collectd/collectd.git ~/collectd
@@ -194,29 +202,6 @@ function setup_agent() {
   sudo systemctl daemon-reload
   sudo systemctl start collectd.service
 
-  log "install VES agent prerequisites"
-  sudo pip install pyyaml
-
-  log "clone OPNFV Barometer"
-  git clone https://gerrit.opnfv.org/gerrit/barometer ~/barometer
-
-  log "setup ves_app_config.conf"
-  cd ~/barometer/3rd_party/collectd-ves-app/ves_app
-  cat <<EOF >ves_app_config.conf
-[config]
-Domain = $ves_host
-Port = $ves_port
-Path = $ves_path
-Topic = $ves_topic
-UseHttps = $ves_https
-Username = $ves_user
-Password = $ves_pass
-SendEventInterval = $ves_interval
-ApiVersion = $ves_version
-KafkaPort = $ves_kafka_port
-KafkaBroker = $ves_kafka_host
-EOF
-
   log "setup VES collectd config for VES $ves_mode mode"
   if [[ "$ves_mode" == "host" ]]; then
     # TODO: Barometer VES guide to clarify prerequisites install for Ubuntu
@@ -228,19 +213,16 @@ EOF
     # http://docs.opnfv.org/en/latest/submodules/barometer/docs/release/userguide/feature.userguide.html#virt-plugin
     sudo systemctl start libvirtd
 
-    git clone https://github.com/maryamtahhan/collectd ~/collectd-virt
-    cd ~/collectd-virt
+    rm -rf /tmp/ves/collectd-virt
+    git clone https://github.com/maryamtahhan/collectd /tmp/ves/collectd-virt
+    cd /tmp/ves/collectd-virt
     ./build.sh
     ./configure --enable-syslog --enable-logfile --enable-debug
     make
     sudo make install
 
-
-    # TODO: Barometer VES guide refers to "At least one VM instance should be 
-    # up and running by hypervisor on the host." The process needs to accomodate
-    # pre-installation of the VES agent *prior* to the first VM being created. 
-
     cat <<EOF | sudo tee -a /opt/collectd/etc/collectd.conf
+# for VES plugin
 LoadPlugin logfile
 <Plugin logfile>
   LogLevel info
@@ -270,6 +252,7 @@ LoadPlugin write_kafka
 EOF
   else
     cat <<EOF | sudo tee -a /opt/collectd/etc/collectd.conf
+# for VES plugin
 LoadPlugin logfile
 <Plugin logfile>
   LogLevel info
@@ -280,15 +263,6 @@ LoadPlugin logfile
 
 LoadPlugin cpu
 
-#LoadPlugin virt
-#<Plugin virt>
-#  Connection "qemu:///system"
-#  RefreshInterval 60
-#  HostnameFormat uuid
-#  PluginInstanceFormat name
-#  ExtraStats "cpu_util"
-#</Plugin>
-
 LoadPlugin write_kafka
 <Plugin write_kafka>
   Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port"
@@ -299,14 +273,61 @@ LoadPlugin write_kafka
 EOF
   fi
 
+  if [[ $(grep -c $ves_hostname /etc/hosts) -eq 0 ]]; then
+    log "add to /etc/hosts: $ves_kafka_host $ves_hostname"
+    echo "$ves_kafka_host $ves_hostname" | sudo tee -a /etc/hosts
+
   log "restart collectd to apply updated config"
   sudo systemctl restart collectd
+}
 
-  log "start VES agent"
-  cd ~/barometer/3rd_party/collectd-ves-app/ves_app
-  nohup python ves_app.py \
-    --events-schema=$ves_mode.yaml \
-    --config=ves_app_config.conf > ~/ves_app.stdout.log 2>&1 &
+function setup_agent() {
+  log "setup VES agent"
+  if [[ ! -f /.dockerenv ]]; then
+    log "start the ves-agent container"
+    sudo docker run -it -d -v /tmp/ves:/opt/ves --name=ves-agent \
+    ubuntu:xenial /bin/bash 
+    log "execute the agent setup script in the container"
+    sudo docker exec ves-agent /bin/bash /opt/ves/ves-setup.sh agent
+  else
+    common_prereqs
+    log "setup the VES environment"
+    source /opt/ves/ves_env.sh
+    log "install agent prerequisites"
+    pip install pyaml
+
+    setup_kafka_client
+
+    log "clone OPNFV Barometer"
+    rm -rf /opt/ves/barometer
+    git clone https://gerrit.opnfv.org/gerrit/barometer /opt/ves/barometer
+
+    log "setup ves_app_config.conf"
+    source /opt/ves/ves_env.sh
+    cd /opt/ves/barometer/3rd_party/collectd-ves-app/ves_app
+    cat <<EOF >ves_app_config.conf
+[config]
+Domain = $ves_host
+Port = $ves_port
+Path = $ves_path
+Topic = $ves_topic
+UseHttps = $ves_https
+Username = $ves_user
+Password = $ves_pass
+SendEventInterval = $ves_interval
+ApiVersion = $ves_version
+KafkaPort = $ves_kafka_port
+KafkaBroker = $ves_kafka_host
+EOF
+
+#    log "add guest.yaml measurements to host.yaml (enables actual host data)"
+#    tail --lines=+24 guest.yaml >>host.yaml
+
+    log "start VES agent"
+    echo "$ves_kafka_host $ves_hostname">>/etc/hosts
+    nohup python ves_app.py --events-schema=$ves_mode.yaml --loglevel ERROR \
+      --config=ves_app_config.conf > /opt/ves/ves_app.stdout.log 2>&1 &
+  fi
 }
 
 function setup_collector() {
@@ -314,29 +335,19 @@ function setup_collector() {
   log "install prerequistes"
   sudo apt-get install -y  jq
 
+  ves_hostname=$HOSTNAME
+  export ves_hostname
   ves_host=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
   export ves_host
   setup_env
 
-  echo "cleanup any earlier install attempts"
-  sudo docker stop influxdb
-  sudo docker rm influxdb
-  sudo docker stop grafana
-  sudo docker rm grafana
-  sudo docker stop ves-collector
-  sudo docker rm -v ves-collector
-  sudo rm -rf /tmp/ves
-
-  log "clone OPNFV VES"
-  git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves
-
   log "setup influxdb container"
-  sudo docker run -d --name=influxdb -p 8086:8086 influxdb
-  status=$(sudo docker inspect influxdb | jq -r '.[0].State.Status')
+  sudo docker run -d --name=ves-influxdb -p 8086:8086 influxdb
+  status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status')
   while [[ "x$status" != "xrunning" ]]; do
     log "InfluxDB container state is ($status)"
     sleep 10
-    status=$(sudo docker inspect influxdb | jq -r '.[0].State.Status')
+    status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status')
   done
   log "InfluxDB container state is $status"
 
@@ -351,23 +362,23 @@ function setup_collector() {
     --data-urlencode "q=CREATE DATABASE veseventsdb"
 
   log "install Grafana container"
-  sudo docker run -d --name grafana -p 3000:3000 grafana/grafana
-  status=$(sudo docker inspect grafana | jq -r '.[0].State.Status')
+  sudo docker run -d --name ves-grafana -p 3001:3000 grafana/grafana
+  status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status')
   while [[ "x$status" != "xrunning" ]]; do
     log "Grafana container state is ($status)"
     sleep 10
-    status=$(sudo docker inspect grafana | jq -r '.[0].State.Status')
+    status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status')
   done
   log "Grafana container state is $status"
 
   log "wait for Grafana API to be active"
-  while ! curl http://$ves_host:3000 ; do
+  while ! curl http://$ves_host:3001 ; do
     log "Grafana API is not yet responding... waiting 10 seconds"
     sleep 10
   done
 
   log "add VESEvents datasource to Grafana"
-  cat <<EOF >datasource.json
+  cat <<EOF >/tmp/ves/datasource.json
 { "name":"VESEvents",
   "type":"influxdb",
   "access":"direct",
@@ -385,18 +396,19 @@ function setup_collector() {
 EOF
 
   curl -H "Accept: application/json" -H "Content-type: application/json" \
-    -X POST -d @datasource.json \
-    http://admin:admin@$ves_host:3000/api/datasources
+    -X POST -d @/tmp/ves/datasource.json \
+    http://admin:admin@$ves_host:3001/api/datasources
 
   log "add VES dashboard to Grafana"
   curl -H "Accept: application/json" -H "Content-type: application/json" \
     -X POST \
-    -d @/tmp/ves/tests/onap-demo/blueprints/tosca-vnfd-onap-demo/Dashboard.json\
-    http://admin:admin@$ves_host:3000/api/dashboards/db	
+    -d @/tmp/ves/tools/grafana/Dashboard.json\
+    http://admin:admin@$ves_host:3001/api/dashboards/db	
 
   log "setup collector container"
   cd /tmp/ves
   touch monitor.log
+  rm -rf /tmp/ves/evel-test-collector
   git clone https://github.com/att/evel-test-collector.git
   sed -i -- \
     "s~log_file = /var/log/att/collector.log~log_file = /opt/ves/collector.log~" \
@@ -411,8 +423,10 @@ EOF
     evel-test-collector/config/collector.conf
   sed -i -- "s~vel_topic_name = example_vnf~vel_topic_name = $ves_topic~g" \
     evel-test-collector/config/collector.conf
+  sed -i -- "/vel_topic_name = /a influxdb = $ves_host" \
+    evel-test-collector/config/collector.conf
 
-  cp tests/onap-demo/blueprints/tosca-vnfd-onap-demo/monitor.py \
+  cp /tmp/ves/tools/monitor.py \
     evel-test-collector/code/collector/monitor.py
 
   # Note below: python (2.7) is required due to dependency on module 'ConfigParser'
@@ -424,28 +438,46 @@ pip install requests
 python /opt/ves/evel-test-collector/code/collector/monitor.py \
 --config /opt/ves/evel-test-collector/config/collector.conf \
 --influxdb $ves_host \
---section default > /opt/ves/monitor.log
+--section default > /opt/ves/monitor.log 2>&1 &
 EOF
 
   sudo docker run -it -d -v /tmp/ves:/opt/ves --name=ves-collector \
     -p 30000:30000 ubuntu:xenial /bin/bash
-  sudo docker exec -it -d ves-collector bash /opt/ves/setup-collector.sh
+  sudo docker exec ves-collector /bin/bash /opt/ves/setup-collector.sh
   # debug hints
   # sudo docker exec -it ves-collector apt-get install -y tcpdump
   # sudo docker exec -it ves-collector tcpdump -A -v -s 0 -i any port 30000
   # curl http://$ves_host:30000
   # sudo docker exec -it ves-collector /bin/bash
-  # ~/kafka_2.11-0.11.0.0/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic collectd
+  # ~/kafka_2.11-0.11.0.2/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic collectd
+}
+
+function clean() {
+  log "clean installation for $1 at $2"
+  if [[ "$1" == "master" ]]; then
+    cs="ves-agent ves-collector ves-grafana ves-influxdb"
+    for c in $cs; do
+      log "stop and remove container $c"
+      sudo docker stop $c
+      sudo docker rm -v $c
+    done
+  fi
+  log "remove collectd config for VES"
+  sudo sed -i -- '/VES plugin/,$d' /opt/collectd/etc/collectd.conf
+  sudo systemctl restart collectd
+  sudo rm -rf /tmp/ves
 }
 
 dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'`
-setup_env
 if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then 
   echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" |\
     sudo tee -a /etc/hosts
 fi
 
 case "$1" in
+  "collectd")
+    setup_collectd
+    ;;
   "agent")
     setup_agent
     ;;
@@ -455,6 +487,9 @@ case "$1" in
   "kafka")
     setup_kafka 
     ;;
+  "clean")
+    clean $2 $3
+    ;;
   *)
     grep '#. ' $0
 esac