From 4b0b335d54946cbb202dfdf0545d499cf559faaa Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Wed, 15 Nov 2017 16:27:17 -0800 Subject: Updated monitor.py and grafana dashboard JIRA: VES-2 Various fixes in ves-setup Add demo_deploy.sh Refactored to containerized agent and collector Change-Id: I8851465742aaf40a4cce265508a3d2d66abced08 Signed-off-by: Bryan Sullivan --- tools/ves-setup.sh | 279 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 157 insertions(+), 122 deletions(-) (limited to 'tools/ves-setup.sh') diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh index 65dd613..b37662b 100644 --- a/tools/ves-setup.sh +++ b/tools/ves-setup.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#. What this is: Setup script for VES agent framework. +#. What this is: Setup script for the VES monitoring framework. #. With this script VES support can be installed in one or more hosts, with: #. - a dedicated or shared Kafka server for collection of events from collectd #. - VES collectd agents running in host or guest mode @@ -23,6 +23,7 @@ #. pre-installed VES collector e.g. from the ONAP project. #. - Install Kafka server on one of the hosts, or use a pre-installed server #. accessible from the agent hosts. +#. - Install collectd on each host. #. - Install the VES agent on each host. #. - As needed, install the VES agent on each virtual host. This could include #. pre-installed VES agents in VM or container images, which are configured @@ -47,21 +48,13 @@ #. ves_kafka_host: kafka host IP or hostname (default: 127.0.0.1) #. #. Usage: -#. wget https://raw.githubusercontent.com/opnfv/ves/master/tools/ves-setup.sh -#. bash ves-setup.sh +#. git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves +#. bash /tmp/ves/ves-setup.sh #. collector: setup VES collector (test collector) #. kafka: setup kafka server for VES events from collect agent(s) -#. agent: setup VES agent in host or guest mode -#. Recommended sequence is: -#. ssh into your collector host and run these commands: -#. $ ves_host=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') -#. $ export ves_host -#. $ bash ves-setup.sh collector -#. ...then for each agent host: -#. copy ~/ves_env.sh and ves-setup.sh to the host e.g. via scp -#. ssh into the host and run, directly or via ssh -x -#. $ bash ves-setup.sh kafka -#. $ bash ves-setup.sh agent +#. collectd: setup collectd with libvirt plugin, as a kafka publisher +#. agent: setup VES agent in host or guest mode, as a kafka consumer +#. See demo_deploy.sh in this repo for a recommended sequence of the above. #. #. Status: this is a work in progress, under test. @@ -81,18 +74,27 @@ function log() { function common_prereqs() { log "install common prerequisites" - sudo apt-get install -y default-jre - sudo apt-get install -y zookeeperd - sudo apt-get install -y python-pip - sudo pip install kafka-python + if [[ ! -f /.dockerenv ]]; then dosudo="sudo"; fi + $dosudo apt-get update + $dosudo apt-get install -y git + # Required for kafka + $dosudo apt-get install -y default-jre + $dosudo apt-get install -y zookeeperd + $dosudo apt-get install -y python-pip + $dosudo pip install kafka-python + # Required for building collectd + $dosudo apt-get install -y pkg-config } function setup_env() { - if [[ ! -f ~/ves_env.sh ]]; then - cat <~/ves_env.sh + if [[ ! -d /tmp/ves ]]; then mkdir /tmp/ves; fi + cp $0 /tmp/ves + if [[ ! -f /tmp/ves/ves_env.sh ]]; then + cat </tmp/ves/ves_env.sh #!/bin/bash ves_mode="${ves_mode:=host}" ves_host="${ves_host:=127.0.0.1}" +ves_hostname="${ves_hostname:=localhost}" ves_port="${ves_port:=30000}" ves_path="${ves_path:=}" ves_topic="${ves_topic:=}" @@ -105,6 +107,7 @@ ves_kafka_host="${ves_kafka_host:=127.0.0.1}" ves_kafka_port="${ves_kafka_port:=9092}" export ves_mode export ves_host +export ves_hostname export ves_port export ves_path export ves_topic @@ -112,57 +115,62 @@ export ves_https export ves_user export ves_pass export ves_interval -export ves_kafka_host -export ves_port +export ves_kafka_hostame export ves_kafka_port EOF fi - source ~/ves_env.sh + source /tmp/ves/ves_env.sh } function setup_kafka() { log "setup kafka server" common_prereqs + setup_env - log "get and unpack kafka_2.11-0.11.0.0.tgz" - wget "http://www-eu.apache.org/dist/kafka/0.11.0.0/kafka_2.11-0.11.0.0.tgz" - tar -xvzf kafka_2.11-0.11.0.0.tgz + cd + ver="0.11.0.2" + log "get and unpack kafka_2.11-$ver.tgz" + wget "http://www-eu.apache.org/dist/kafka/$ver/kafka_2.11-$ver.tgz" + tar -xvzf kafka_2.11-$ver.tgz log "set delete.topic.enable=true" sed -i -- 's/#delete.topic.enable=true/delete.topic.enable=true/' \ - kafka_2.11-0.11.0.0/config/server.properties - grep delete.topic.enable kafka_2.11-0.11.0.0/config/server.properties + kafka_2.11-$ver/config/server.properties + grep delete.topic.enable kafka_2.11-$ver/config/server.properties # TODO: Barometer VES guide to clarify hostname must be in /etc/hosts - sudo nohup kafka_2.11-0.11.0.0/bin/kafka-server-start.sh \ - kafka_2.11-0.11.0.0/config/server.properties \ - > kafka_2.11-0.11.0.0/kafka.log 2>&1 & - # TODO: find a test that does not hang the script at - # echo "Hello, World" | ~/kafka_2.11-0.11.0.0/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic TopicTest > /dev/null - # ~/kafka_2.11-0.11.0.0/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic TopicTest --from-beginning + sudo nohup kafka_2.11-$ver/bin/kafka-server-start.sh \ + kafka_2.11-$ver/config/server.properties \ + > kafka_2.11-$ver/kafka.log 2>&1 & } -function setup_agent() { - log "setup agent" - common_prereqs - - log "cleanup any previous failed install" - sudo rm -rf ~/collectd-virt - sudo rm -rf ~/librdkafka - sudo rm -rf ~/collectd - +function setup_kafka_client() { log "Install Apache Kafka C/C++ client library" - sudo apt-get install -y build-essential + if [[ ! -f /.dockerenv ]]; then dosudo="sudo"; fi + $dosudo apt-get install -y build-essential git clone https://github.com/edenhill/librdkafka.git ~/librdkafka cd ~/librdkafka git checkout -b v0.9.5 v0.9.5 # TODO: Barometer VES guide to clarify specific prerequisites for Ubuntu - sudo apt-get install -y libpthread-stubs0-dev - sudo apt-get install -y libssl-dev - sudo apt-get install -y libsasl2-dev - sudo apt-get install -y liblz4-dev + $dosudo apt-get install -y libpthread-stubs0-dev + $dosudo apt-get install -y libssl-dev + $dosudo apt-get install -y libsasl2-dev + $dosudo apt-get install -y liblz4-dev ./configure --prefix=/usr make - sudo make install + $dosudo make install +} + +function setup_collectd() { + log "setup collectd" + common_prereqs + setup_env + + log "cleanup any previous failed install" + sudo rm -rf ~/collectd-virt + sudo rm -rf ~/librdkafka + sudo rm -rf ~/collectd + + setup_kafka_client log "Build collectd with Kafka support" git clone https://github.com/collectd/collectd.git ~/collectd @@ -194,29 +202,6 @@ function setup_agent() { sudo systemctl daemon-reload sudo systemctl start collectd.service - log "install VES agent prerequisites" - sudo pip install pyyaml - - log "clone OPNFV Barometer" - git clone https://gerrit.opnfv.org/gerrit/barometer ~/barometer - - log "setup ves_app_config.conf" - cd ~/barometer/3rd_party/collectd-ves-app/ves_app - cat <ves_app_config.conf -[config] -Domain = $ves_host -Port = $ves_port -Path = $ves_path -Topic = $ves_topic -UseHttps = $ves_https -Username = $ves_user -Password = $ves_pass -SendEventInterval = $ves_interval -ApiVersion = $ves_version -KafkaPort = $ves_kafka_port -KafkaBroker = $ves_kafka_host -EOF - log "setup VES collectd config for VES $ves_mode mode" if [[ "$ves_mode" == "host" ]]; then # TODO: Barometer VES guide to clarify prerequisites install for Ubuntu @@ -228,19 +213,16 @@ EOF # http://docs.opnfv.org/en/latest/submodules/barometer/docs/release/userguide/feature.userguide.html#virt-plugin sudo systemctl start libvirtd - git clone https://github.com/maryamtahhan/collectd ~/collectd-virt - cd ~/collectd-virt + rm -rf /tmp/ves/collectd-virt + git clone https://github.com/maryamtahhan/collectd /tmp/ves/collectd-virt + cd /tmp/ves/collectd-virt ./build.sh ./configure --enable-syslog --enable-logfile --enable-debug make sudo make install - - # TODO: Barometer VES guide refers to "At least one VM instance should be - # up and running by hypervisor on the host." The process needs to accomodate - # pre-installation of the VES agent *prior* to the first VM being created. - cat < LogLevel info @@ -270,6 +252,7 @@ LoadPlugin write_kafka EOF else cat < LogLevel info @@ -280,15 +263,6 @@ LoadPlugin logfile LoadPlugin cpu -#LoadPlugin virt -# -# Connection "qemu:///system" -# RefreshInterval 60 -# HostnameFormat uuid -# PluginInstanceFormat name -# ExtraStats "cpu_util" -# - LoadPlugin write_kafka Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port" @@ -299,14 +273,61 @@ LoadPlugin write_kafka EOF fi + if [[ $(grep -c $ves_hostname /etc/hosts) -eq 0 ]]; then + log "add to /etc/hosts: $ves_kafka_host $ves_hostname" + echo "$ves_kafka_host $ves_hostname" | sudo tee -a /etc/hosts + log "restart collectd to apply updated config" sudo systemctl restart collectd +} - log "start VES agent" - cd ~/barometer/3rd_party/collectd-ves-app/ves_app - nohup python ves_app.py \ - --events-schema=$ves_mode.yaml \ - --config=ves_app_config.conf > ~/ves_app.stdout.log 2>&1 & +function setup_agent() { + log "setup VES agent" + if [[ ! -f /.dockerenv ]]; then + log "start the ves-agent container" + sudo docker run -it -d -v /tmp/ves:/opt/ves --name=ves-agent \ + ubuntu:xenial /bin/bash + log "execute the agent setup script in the container" + sudo docker exec ves-agent /bin/bash /opt/ves/ves-setup.sh agent + else + common_prereqs + log "setup the VES environment" + source /opt/ves/ves_env.sh + log "install agent prerequisites" + pip install pyaml + + setup_kafka_client + + log "clone OPNFV Barometer" + rm -rf /opt/ves/barometer + git clone https://gerrit.opnfv.org/gerrit/barometer /opt/ves/barometer + + log "setup ves_app_config.conf" + source /opt/ves/ves_env.sh + cd /opt/ves/barometer/3rd_party/collectd-ves-app/ves_app + cat <ves_app_config.conf +[config] +Domain = $ves_host +Port = $ves_port +Path = $ves_path +Topic = $ves_topic +UseHttps = $ves_https +Username = $ves_user +Password = $ves_pass +SendEventInterval = $ves_interval +ApiVersion = $ves_version +KafkaPort = $ves_kafka_port +KafkaBroker = $ves_kafka_host +EOF + +# log "add guest.yaml measurements to host.yaml (enables actual host data)" +# tail --lines=+24 guest.yaml >>host.yaml + + log "start VES agent" + echo "$ves_kafka_host $ves_hostname">>/etc/hosts + nohup python ves_app.py --events-schema=$ves_mode.yaml --loglevel ERROR \ + --config=ves_app_config.conf > /opt/ves/ves_app.stdout.log 2>&1 & + fi } function setup_collector() { @@ -314,29 +335,19 @@ function setup_collector() { log "install prerequistes" sudo apt-get install -y jq + ves_hostname=$HOSTNAME + export ves_hostname ves_host=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') export ves_host setup_env - echo "cleanup any earlier install attempts" - sudo docker stop influxdb - sudo docker rm influxdb - sudo docker stop grafana - sudo docker rm grafana - sudo docker stop ves-collector - sudo docker rm -v ves-collector - sudo rm -rf /tmp/ves - - log "clone OPNFV VES" - git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves - log "setup influxdb container" - sudo docker run -d --name=influxdb -p 8086:8086 influxdb - status=$(sudo docker inspect influxdb | jq -r '.[0].State.Status') + sudo docker run -d --name=ves-influxdb -p 8086:8086 influxdb + status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status') while [[ "x$status" != "xrunning" ]]; do log "InfluxDB container state is ($status)" sleep 10 - status=$(sudo docker inspect influxdb | jq -r '.[0].State.Status') + status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status') done log "InfluxDB container state is $status" @@ -351,23 +362,23 @@ function setup_collector() { --data-urlencode "q=CREATE DATABASE veseventsdb" log "install Grafana container" - sudo docker run -d --name grafana -p 3000:3000 grafana/grafana - status=$(sudo docker inspect grafana | jq -r '.[0].State.Status') + sudo docker run -d --name ves-grafana -p 3001:3000 grafana/grafana + status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') while [[ "x$status" != "xrunning" ]]; do log "Grafana container state is ($status)" sleep 10 - status=$(sudo docker inspect grafana | jq -r '.[0].State.Status') + status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') done log "Grafana container state is $status" log "wait for Grafana API to be active" - while ! curl http://$ves_host:3000 ; do + while ! curl http://$ves_host:3001 ; do log "Grafana API is not yet responding... waiting 10 seconds" sleep 10 done log "add VESEvents datasource to Grafana" - cat <datasource.json + cat </tmp/ves/datasource.json { "name":"VESEvents", "type":"influxdb", "access":"direct", @@ -385,18 +396,19 @@ function setup_collector() { EOF curl -H "Accept: application/json" -H "Content-type: application/json" \ - -X POST -d @datasource.json \ - http://admin:admin@$ves_host:3000/api/datasources + -X POST -d @/tmp/ves/datasource.json \ + http://admin:admin@$ves_host:3001/api/datasources log "add VES dashboard to Grafana" curl -H "Accept: application/json" -H "Content-type: application/json" \ -X POST \ - -d @/tmp/ves/tests/onap-demo/blueprints/tosca-vnfd-onap-demo/Dashboard.json\ - http://admin:admin@$ves_host:3000/api/dashboards/db + -d @/tmp/ves/tools/grafana/Dashboard.json\ + http://admin:admin@$ves_host:3001/api/dashboards/db log "setup collector container" cd /tmp/ves touch monitor.log + rm -rf /tmp/ves/evel-test-collector git clone https://github.com/att/evel-test-collector.git sed -i -- \ "s~log_file = /var/log/att/collector.log~log_file = /opt/ves/collector.log~" \ @@ -411,8 +423,10 @@ EOF evel-test-collector/config/collector.conf sed -i -- "s~vel_topic_name = example_vnf~vel_topic_name = $ves_topic~g" \ evel-test-collector/config/collector.conf + sed -i -- "/vel_topic_name = /a influxdb = $ves_host" \ + evel-test-collector/config/collector.conf - cp tests/onap-demo/blueprints/tosca-vnfd-onap-demo/monitor.py \ + cp /tmp/ves/tools/monitor.py \ evel-test-collector/code/collector/monitor.py # Note below: python (2.7) is required due to dependency on module 'ConfigParser' @@ -424,28 +438,46 @@ pip install requests python /opt/ves/evel-test-collector/code/collector/monitor.py \ --config /opt/ves/evel-test-collector/config/collector.conf \ --influxdb $ves_host \ ---section default > /opt/ves/monitor.log +--section default > /opt/ves/monitor.log 2>&1 & EOF sudo docker run -it -d -v /tmp/ves:/opt/ves --name=ves-collector \ -p 30000:30000 ubuntu:xenial /bin/bash - sudo docker exec -it -d ves-collector bash /opt/ves/setup-collector.sh + sudo docker exec ves-collector /bin/bash /opt/ves/setup-collector.sh # debug hints # sudo docker exec -it ves-collector apt-get install -y tcpdump # sudo docker exec -it ves-collector tcpdump -A -v -s 0 -i any port 30000 # curl http://$ves_host:30000 # sudo docker exec -it ves-collector /bin/bash - # ~/kafka_2.11-0.11.0.0/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic collectd + # ~/kafka_2.11-0.11.0.2/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic collectd +} + +function clean() { + log "clean installation for $1 at $2" + if [[ "$1" == "master" ]]; then + cs="ves-agent ves-collector ves-grafana ves-influxdb" + for c in $cs; do + log "stop and remove container $c" + sudo docker stop $c + sudo docker rm -v $c + done + fi + log "remove collectd config for VES" + sudo sed -i -- '/VES plugin/,$d' /opt/collectd/etc/collectd.conf + sudo systemctl restart collectd + sudo rm -rf /tmp/ves } dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'` -setup_env if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" |\ sudo tee -a /etc/hosts fi case "$1" in + "collectd") + setup_collectd + ;; "agent") setup_agent ;; @@ -455,6 +487,9 @@ case "$1" in "kafka") setup_kafka ;; + "clean") + clean $2 $3 + ;; *) grep '#. ' $0 esac -- cgit 1.2.3-korg