diff options
Diffstat (limited to 'tools/ves-setup.sh')
| -rw-r--r-- | tools/ves-setup.sh | 611 |
1 files changed, 254 insertions, 357 deletions
diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh index 00d7db8..8d5c902 100644 --- a/tools/ves-setup.sh +++ b/tools/ves-setup.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2017 AT&T Intellectual Property, Inc +# Copyright 2017-2018 AT&T Intellectual Property, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +15,8 @@ # #. What this is: Setup script for the VES monitoring framework. #. With this script VES support can be installed in one or more hosts, with: -#. - a dedicated or shared Kafka server for collection of events from collectd -#. - VES collectd agents running in host or guest mode +#. - a dedicated or shared Kafka server for collection of events from barometer +#. - VES barometer agents running in host or guest mode #. - VES monitor (test collector) #. - Influxdb service (if an existing service is not passed as an option) #. - Grafana service (if an existing service is not passed as an option) @@ -26,22 +26,22 @@ #. pre-installed VES collector e.g. from the ONAP project. #. - Install Kafka server on one of the hosts, or use a pre-installed server #. accessible from the agent hosts. -#. - Install collectd on each host. +#. - Install barometer on each host. #. - Install the VES agent on one of the hosts. #. #. Prerequisites: #. - Ubuntu Xenial (Centos support to be provided) #. - passwordless sudo setup for user running this script #. - shell environment variables setup as below (for non-default setting) -#. ves_mode: install mode (node|guest) for VES collectd plugin (default: node) +#. ves_mode: install mode (node|guest) for VES barometer plugin (default: node) #. ves_host: VES collector IP or hostname (default: 127.0.0.1) -#. ves_port: VES collector port (default: 30000) +#. ves_port: VES collector port (default: 3001) #. ves_path: REST path optionalRoutingPath element (default: empty) #. ves_topic: REST path topicName element (default: empty) #. ves_https: use HTTPS instead of HTTP (default: false) #. ves_user: username for basic auth with collector (default: empty) #. ves_pass: password for basic auth with collector (default: empty) -#. ves_interval: frequency in sec for collectd data reports (default: 20) +#. ves_interval: frequency in sec for barometer data reports (default: 20) #. ves_version: VES API version (default: 5.1) #. ves_kafka_host: kafka host IP (default: 127.0.0.1) #. ves_kafka_hostname: kafka host hostname (default: localhost) @@ -54,18 +54,28 @@ #. #. Usage: #. git clone https://gerrit.opnfv.org/gerrit/ves ~/ves -#. bash ~/ves/tools/ves-setup.sh <collector|kafka|collectd|agent> [cloudify] +#. bash ~/ves/tools/ves-setup.sh <what> [cloudify] +#. what: one of env|influxdb|grafana|collector|zookeeper|kafka|agent|barometer +#. env: setup VES environment script ~/ves/tools/ves_env.sh +#. influxdb: setup influxdb as a docker container on k8s_master node +#. grafana: setup grafana as a docker container on k8s_master node #. collector: setup VES collector (test collector) +#. zookeeper: setup zookeeper server for kafka configuration #. kafka: setup kafka server for VES events from collect agent(s) -#. collectd: setup collectd with libvirt plugin, as a kafka publisher #. agent: setup VES agent in host or guest mode, as a kafka consumer +#. barometer: setup barometer with libvirt plugin, as a kafka publisher #. cloudify: (optional) use cloudify to deploy the component, as setup by #. tools/cloudify/k8s-cloudify.sh in the OPNFV Models repo. -#. bash ~/ves/ves-setup.sh <master> <workers> -#. master: VES master node IP -#. workers: quoted, space-separated list of worker node IPs #. -#. See demo_deploy.sh in this repo for a recommended sequence of the above. +#. The recommended sequence for setting up the components is: +#. influxdb: prerequisite for grafana datasource setup +#. grafana: prerequisite for setup of datasource and dashboards +#. collector: creates veseventsdb in influxdb, and grafana +#. datasource/dashboards, then starts listening for VES event reports +#. zookeeper: prerequisite for kafka +#. kafka: prerequisite for agent and barometer +#. agent: listens for collectd topic events over kafka, for reporting to collector +#. barometer: monitors resources and reports via collectd topic in kafka #. #. Status: this is a work in progress, under test. @@ -95,11 +105,13 @@ function common_prereqs() { } function setup_env() { - cat <<'EOF' >~/ves/tools/ves_env.sh + log "updating VES environment variables" + cat <<EOF >~/ves/tools/ves_env.sh #!/bin/bash ves_mode="${ves_mode:=node}" -ves_host="${ves_host:=127.0.0.1}" -ves_port="${ves_port:=30000}" +ves_host="${ves_host:=ves-collector-service.default.svc.cluster.local}" +ves_hostname="${ves_hostname:=ves-collector-service.default.svc.cluster.local}" +ves_port="${ves_port:=3001}" ves_path="${ves_path:=}" ves_topic="${ves_topic:=}" ves_https="${ves_https:=false}" @@ -107,17 +119,26 @@ ves_user="${ves_user:=}" ves_pass="${ves_pass:=}" ves_interval="${ves_interval:=20}" ves_version="${ves_version:=5.1}" -ves_kafka_host="${ves_kafka_host:=127.0.0.1}" -ves_kafka_hostname="${ves_kafka_hostname:=localhost}" +ves_zookeeper_host="${ves_zookeeper_host:=ves-zookeeper-service.default.svc.cluster.local}" +ves_zookeeper_hostname="${ves_zookeeper_hostname:=ves-zookeeper-service.default.svc.cluster.local}" +ves_zookeeper_host="${ves_zookeeper_host:=ves-zookeeper-service.default.svc.cluster.local}" +ves_zookeeper_port="${ves_zookeeper_port:=2181}" +ves_kafka_host="${ves_kafka_host:=ves-kafka-service.default.svc.cluster.local}" +ves_kafka_hostname="${ves_kafka_hostname:=ves-kafka-service.default.svc.cluster.local}" ves_kafka_port="${ves_kafka_port:=9092}" -ves_influxdb_host="${ves_influxdb_host:=localhost:8086}" +ves_influxdb_host="${ves_influxdb_host:=ves-influxdb-service.default.svc.cluster.local}" +ves_influxdb_hostname="${ves_influxdb_hostname:=ves-influxdb-service.default.svc.cluster.local}" +ves_influxdb_port="${ves_influxdb_port:=8086}" ves_influxdb_auth="${ves_influxdb_auth:=}" -ves_grafana_host="${ves_grafana_host:=localhost:3000}" +ves_grafana_host="${ves_grafana_host:=ves-grafana-service.default.svc.cluster.local}" +ves_grafana_hostname="${ves_grafana_hostname:=ves-grafana-service.default.svc.cluster.local}" +ves_grafana_port="${ves_grafana_port:=3000}" ves_grafana_auth="${ves_grafana_auth:=admin:admin}" -ves_loglevel="${ves_loglevel:=}" +ves_loglevel="${ves_loglevel:=DEBUG}" ves_cloudtype="${ves_cloudtype:=kubernetes}" export ves_mode export ves_host +export ves_hostname export ves_port export ves_path export ves_topic @@ -125,187 +146,105 @@ export ves_https export ves_user export ves_pass export ves_interval +export ves_version +export ves_zookeeper_host +export ves_zookeeper_hostname +export ves_zookeeper_port export ves_kafka_host export ves_kafka_hostname export ves_kafka_port export ves_influxdb_host +export ves_influxdb_hostname +export ves_influxdb_port export ves_influxdb_auth export ves_grafana_host +export ves_grafana_hostname +export ves_grafana_port export ves_grafana_auth export ves_loglevel export ves_cloudtype EOF source ~/ves/tools/ves_env.sh - echo ~/ves/tools/ves_env.sh + env | grep ves_ } -function setup_collectd() { - log "setup collectd" - - common_prereqs - source ~/ves/tools/ves_env.sh - - log "setup VES collectd config for VES $ves_mode mode" - mkdir ~/collectd - if [[ "$ves_mode" == "node" ]]; then -# # TODO: fix for journalctl -xe report "... is marked executable" -# sudo chmod 744 /etc/systemd/system/collectd.service - - cat <<EOF >~/collectd/collectd.conf -# for VES plugin -LoadPlugin logfile -<Plugin logfile> - LogLevel debug - File STDOUT - Timestamp true - PrintSeverity false -</Plugin> - -LoadPlugin csv -<Plugin csv> - DataDir "/work-dir/collectd/install/var/lib/csv" - StoreRates false -</Plugin> - -LoadPlugin target_set -LoadPlugin match_regex -<Chain "PreCache"> - <Rule "mark_memory_as_host"> - <Match "regex"> - Plugin "^memory$" - </Match> - <Target "set"> - PluginInstance "host" - </Target> - </Rule> -</Chain> - -LoadPlugin cpu -<Plugin cpu> - ReportByCpu true - ReportByState true - ValuesPercentage true -</Plugin> - -LoadPlugin interface -LoadPlugin memory -LoadPlugin load -LoadPlugin disk -# TODO: how to set this option only to apply to VMs (not nodes) -LoadPlugin uuid +function update_env() { + log "update VES environment with $1=$2" + eval ${1}=${2} + export $1 + sed -i -- "s/.*$1=.*/$1=$2/" ~/ves/tools/ves_env.sh + env | grep ves_ +} -LoadPlugin write_kafka -<Plugin write_kafka> - Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port" - <Topic "collectd"> - Format JSON - </Topic> -</Plugin> -EOF +function setup_kafka() { + log "setup kafka server" + log "deploy zookeeper and kafka" + if [[ "$1" == "cloudify" ]]; then + cp -r ~/ves/tools/cloudify/ves-zookeeper ~/models/tools/cloudify/blueprints/. + source ~/models/tools/cloudify/k8s-cloudify.sh start ves-zookeeper ves-zookeeper + source ~/models/tools/cloudify/k8s-cloudify.sh clusterIp ves-zookeeper + update_env ves_zookeeper_host $clusterIp - if [[ -d /etc/nova ]]; then - cat <<EOF >>~/collectd/collectd.conf -LoadPlugin virt -<Plugin virt> - Connection "qemu:///system" - RefreshInterval 60 - HostnameFormat uuid - PluginInstanceFormat name - ExtraStats "cpu_util" -</Plugin> -EOF - fi + cp -r ~/ves/tools/cloudify/ves-kafka ~/models/tools/cloudify/blueprints/. + inputs="{ \ + \"zookeeper_hostname\": \"$ves_zookeeper_hostname\", + \"zookeeper_host\": \"$ves_zookeeper_host\", + \"zookeeper_port\": \"$ves_zookeeper_port\", + \"kafka_port\": \"$ves_kafka_port\", + \"kafka_hostname\": \"$ves_kafka_hostname\"}" + + source ~/models/tools/cloudify/k8s-cloudify.sh start ves-kafka ves-kafka "$inputs" + source ~/models/tools/cloudify/k8s-cloudify.sh clusterIp ves-kafka + update_env ves_kafka_host $clusterIp else - cat <<EOF >~/collectd/collectd.conf -# for VES plugin -LoadPlugin logfile -<Plugin logfile> - LogLevel debug - File STDOUT - Timestamp true - PrintSeverity false -</Plugin> - -LoadPlugin cpu -<Plugin cpu> - ReportByCpu true - ReportByState true - ValuesPercentage true -</Plugin> - -LoadPlugin csv -<Plugin csv> - DataDir "/tmp" -</Plugin> - -LoadPlugin interface -LoadPlugin memory -LoadPlugin load -LoadPlugin disk -LoadPlugin uuid - -LoadPlugin write_kafka -<Plugin write_kafka> - Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port" - <Topic "collectd"> - Format JSON - </Topic> -</Plugin> - -LoadPlugin target_set -LoadPlugin match_regex -<Chain "PreCache"> - <Rule "mark_memory_as_guest"> - <Match "regex"> - Plugin "^memory$" - </Match> - <Target "set"> - PluginInstance "guest" - </Target> - </Rule> -</Chain> + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <<EOF +sudo docker run -it -d -p $ves_zookeeper_port:2181 --name ves-zookeeper zookeeper +sudo docker run -it -d -p $ves_kafka_port:9092 --name ves-kafka \ + -e zookeeper_hostname=$ves_zookeeper_hostname \ + -e kafka_hostname=$ves_kafka_hostname \ + -e zookeeper_host=$ves_zookeeper_host \ + -e zookeeper_port=$ves_zookeeper_port \ + -e kafka_port=$ves_kafka_port \ + -e kafka_hostname=$ves_kafka_hostname \ + blsaws/ves-kafka:latest EOF fi - log "collectd config updated" +} - if [[ $(grep -c $ves_kafka_hostname /etc/hosts) -eq 0 ]]; then - log "add to /etc/hosts: $ves_kafka_host $ves_kafka_hostname" - echo "$ves_kafka_host $ves_kafka_hostname" | sudo tee -a /etc/hosts - fi +function setup_barometer() { + log "setup barometer" +# if [[ $(grep -c $ves_kafka_hostname /etc/hosts) -eq 0 ]]; then +# log "add to /etc/hosts: $ves_kafka_host $ves_kafka_hostname" +# echo "$ves_kafka_host $ves_kafka_hostname" | sudo tee -a /etc/hosts +# fi - log "start Barometer container" - sudo docker run -tid --net=host --name ves-barometer \ - -v ~/collectd:/opt/collectd/etc/collectd.conf.d \ - -v /var/run:/var/run -v /tmp:/tmp --privileged \ - opnfv/barometer:latest /run_collectd.sh + log "start Barometer container as daemonset under kubernetes" + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <<EOF +sed -i -- "s/<ves_mode>/$ves_mode/" \ + /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +sed -i -- "s/<ves_kafka_hostname>/$ves_kafka_hostname/" \ + /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +sed -i -- "s/<ves_kafka_port>/$ves_kafka_port/" \ + /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +kubectl create \ + -f /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +EOF + +# sudo docker run -tid --net=host --name ves-barometer \ +# -v ~/collectd:/opt/collectd/etc/collectd.conf.d \ +# -v /var/run:/var/run -v /tmp:/tmp --privileged \ +# opnfv/barometer:latest /run_collectd.sh } function setup_agent() { log "setup VES agent" - source ~/k8s_env.sh - source ~/ves/tools/ves_env.sh log "deploy the VES agent container" if [[ "$1" == "cloudify" ]]; then - cd ~/ves/tools/cloudify - # Cloudify is deployed on the k8s master node - manager_ip=$k8s_master - log "copy kube config from k8s master for insertion into blueprint" - scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - $k8s_user@$manager_ip:/home/$k8s_user/.kube/config ves-agent/kube.config - - log "package the blueprint" - # CLI: cfy blueprints package -o /tmp/$bp $bp - tar ckf /tmp/blueprint.tar ves-agent - - log "upload the blueprint" - # CLI: cfy blueprints upload -t default_tenant -b $bp /tmp/$bp.tar.gz - curl -s -X PUT -u admin:admin --header 'Tenant: default_tenant' \ - --header "Content-Type: application/octet-stream" -o /tmp/json \ - http://$manager_ip/api/v3.1/blueprints/ves-agent?application_file_name=blueprint.yaml \ - -T /tmp/blueprint.tar - + cp -r ~/ves/tools/cloudify/ves-agent ~/models/tools/cloudify/blueprints/. inputs="{ \ \"ves_mode\": \"$ves_mode\", \"ves_host\": \"$ves_host\", @@ -317,44 +256,32 @@ function setup_agent() { \"ves_pass\": \"$ves_pass\", \"ves_interval\": \"$ves_interval\", \"ves_version\": \"$ves_version\", - \"ves_kafka_port\": \"$ves_kafka_port\", - \"ves_kafka_host\": \"$ves_kafka_host\", \"ves_kafka_hostname\": \"$ves_kafka_hostname\", + \"ves_kafka_host\": \"$ves_kafka_host\", + \"ves_kafka_port\": \"$ves_kafka_port\", \"ves_loglevel\": \"$ves_loglevel\"}" - log "create a deployment for the blueprint" - # CLI: cfy deployments create -t default_tenant -b $bp $bp - curl -s -X PUT -u admin:admin --header 'Tenant: default_tenant' \ - --header "Content-Type: application/json" -o /tmp/json \ - -d "{\"blueprint_id\": \"ves-agent\", \"inputs\": $inputs}" \ - http://$manager_ip/api/v3.1/deployments/ves-agent - sleep 10 - - # CLI: cfy workflows list -d $bp - - log "install the deployment pod and service" - # CLI: cfy executions start install -d $bp - curl -s -X POST -u admin:admin --header 'Tenant: default_tenant' \ - --header "Content-Type: application/json" -o /tmp/json \ - -d "{\"deployment_id\":\"ves-agent\", \"workflow_id\":\"install\"}" \ - http://$manager_ip/api/v3.1/executions + bash ~/models/tools/cloudify/k8s-cloudify.sh start ves-agent ves-agent "$inputs" else - sudo docker run -it -d \ - -e ves_mode=$ves_mode \ - -e ves_host=$ves_host \ - -e ves_port=$ves_port \ - -e ves_path=$ves_path \ - -e ves_topic=$ves_topic \ - -e ves_https=$ves_https \ - -e ves_user=$ves_user \ - -e ves_pass=$ves_pass \ - -e ves_interval=$ves_interval \ - -e ves_version=$ves_version \ - -e ves_kafka_port=$ves_kafka_port \ - -e ves_kafka_host=$ves_kafka_host \ - -e ves_kafka_hostname=$ves_kafka_hostname \ - -e ves_loglevel=$ves_loglevel \ - --name ves-agent blsaws/ves-agent:latest + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <<EOF +sudo docker run -it -d \ + -e ves_mode=$ves_mode \ + -e ves_host=$ves_host \ + -e ves_port=$ves_port \ + -e ves_path=$ves_path \ + -e ves_topic=$ves_topic \ + -e ves_https=$ves_https \ + -e ves_user=$ves_user \ + -e ves_pass=$ves_pass \ + -e ves_interval=$ves_interval \ + -e ves_version=$ves_version \ + -e ves_kafka_port=$ves_kafka_port \ + -e ves_kafka_host=$ves_kafka_host \ + -e ves_kafka_hostname=$ves_kafka_hostname \ + -e ves_loglevel=$ves_loglevel \ + --name ves-agent blsaws/ves-agent:latest +EOF fi # debug hints @@ -366,10 +293,8 @@ function setup_agent() { # ~/kafka_2.11-0.11.0.2/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic collectd } -function setup_collector() { - log "setup collector" - $2 $3 $4 - +function setup_influxdb() { + log "setup influxdb" log "install prerequistes" if [[ "$dist" == "ubuntu" ]]; then sudo apt-get install -y jq @@ -377,152 +302,115 @@ function setup_collector() { sudo yum install -y jq fi - setup_env - - if ! curl http://$ves_influxdb_host/ping ; then - # TODO: migrate to deployment via Helm - log "setup influxdb container" - sudo docker run -d --name=ves-influxdb -p 8086:8086 influxdb - status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status') - while [[ "x$status" != "xrunning" ]]; do - log "InfluxDB container state is ($status)" - sleep 10 - status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status') - done - log "InfluxDB container state is $status" - - log "wait for InfluxDB API to be active" - while ! curl http://$ves_influxdb_host/ping ; do - log "InfluxDB API is not yet responding... waiting 10 seconds" - sleep 10 - done - fi - echo "ves_influxdb_host=$ves_influxdb_host" - - log "setup InfluxDB database" - # TODO: check if pre-existing and skip - curl -X POST http://$ves_influxdb_host/query \ - --data-urlencode "q=CREATE DATABASE veseventsdb" - - if ! curl http://$ves_grafana_host ; then - # TODO: migrate to deployment via Helm - log "install Grafana container" - sudo docker run -d --name ves-grafana -p 3000:3000 grafana/grafana - status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') - while [[ "x$status" != "xrunning" ]]; do - log "Grafana container state is ($status)" - sleep 10 - status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') - done - log "Grafana container state is $status" - echo "ves_grafana_host=$ves_grafana_host" - - log "wait for Grafana API to be active" - while ! curl http://$ves_grafana_host ; do - log "Grafana API is not yet responding... waiting 10 seconds" - sleep 10 - done + log "checking for influxdb at http://$ves_influxdb_host:$ves_influxdb_port/ping" + if ! curl http://$ves_influxdb_host:$ves_influxdb_port/ping ; then + log "install influxdb container on k8s master" + update_env ves_influxdb_host $k8s_master + update_env ves_influxdb_hostname $k8s_master_hostname + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master \ + sudo docker run -d --name=ves-influxdb -p 8086:8086 influxdb + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <<'EOF' +status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status') +while [[ "x$status" != "xrunning" ]]; do + echo; echo "InfluxDB container state is ($status)" + sleep 10 + status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status') +done +echo; echo "InfluxDB container state is $status" +EOF fi +} - log "add VESEvents datasource to Grafana at http://$ves_grafana_auth@$ves_grafana_host" - # TODO: check if pre-existing and skip - cat <<EOF >~/ves/tools/grafana/datasource.json -{ "name":"VESEvents", - "type":"influxdb", - "access":"direct", - "url":"http://$ves_host:8086", - "password":"root", - "user":"root", - "database":"veseventsdb", - "basicAuth":false, - "basicAuthUser":"", - "basicAuthPassword":"", - "withCredentials":false, - "isDefault":false, - "jsonData":null -} +function setup_grafana() { + log "setup grafana" + log "checking for grafana at http://$ves_grafana_host:$ves_grafana_port" + if ! curl http://$ves_grafana_host:$ves_grafana_port ; then + log "install Grafana container on k8s master" + update_env ves_grafana_host $k8s_master + update_env ves_grafana_hostname $k8s_master_hostname + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master \ + sudo docker run -d --name ves-grafana -p 3000:3000 grafana/grafana + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <<'EOF' +status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') +while [[ "x$status" != "xrunning" ]]; do + echo; echo "Grafana container state is ($status)" + sleep 10 +status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') +done +echo; echo "Grafana container state is $status" EOF - - # Use /home/$USER/ instead of ~ with @ - curl -H "Accept: application/json" -H "Content-type: application/json" \ - -X POST -d @/home/$USER/ves/tools/grafana/datasource.json \ - http://$ves_grafana_auth@$ves_grafana_host/api/datasources - - log "add VES dashboard to Grafana at http://$ves_grafana_auth@$ves_grafana_host" - curl -H "Accept: application/json" -H "Content-type: application/json" \ - -X POST \ - -d @/home/$USER/ves/tools/grafana/Dashboard.json\ - http://$ves_grafana_auth@$ves_grafana_host/api/dashboards/db - - log "setup collector container" - # TODO: migrate to deployment via Helm - sudo docker run -it -d -p 30000:30000 \ - -e ves_host=$ves_host \ - -e ves_port=$ves_port \ - -e ves_path=$ves_path \ - -e ves_topic=$ves_topic \ - -e ves_https=$ves_https \ - -e ves_user=$ves_user \ - -e ves_pass=$ves_pass \ - -e ves_interval=$ves_interval \ - -e ves_version=$ves_version \ - -e ves_influxdb_host=$ves_influxdb_host \ - -e ves_loglevel=$ves_loglevel \ - --name ves-collector blsaws/ves-collector:latest - - # debug hints - # curl 'http://172.16.0.5:8086/query?pretty=true&db=veseventsdb&q=SELECT%20moving_average%28%22load-shortterm%22%2C%205%29%20FROM%20%22load%22%20WHERE%20time%20%3E%3D%20now%28%29%20-%205m%20GROUP%20BY%20%22system%22' - # sudo docker logs ves-collector - # sudo docker exec -it ves-collector apt-get install -y tcpdump - # sudo docker exec -it ves-collector tcpdump -A -v -s 0 -i any port 30000 - # curl http://$ves_host:30000 - # sudo docker exec -it ves-collector /bin/bash + fi } -function clean() { - log "clean installation" - master=$1 - workers="$2" - source ~/k8s_env.sh +function setup_collector() { + log "setup collector" + if [[ "$1" == "cloudify" ]]; then + cp -r ~/ves/tools/cloudify/ves-collector ~/models/tools/cloudify/blueprints/. + inputs="{ \ + \"ves_host\": \"$ves_host\", + \"ves_port\": \"$ves_port\", + \"ves_path\": \"$ves_path\", + \"ves_topic\": \"$ves_topic\", + \"ves_https\": \"$ves_https\", + \"ves_user\": \"$ves_user\", + \"ves_pass\": \"$ves_pass\", + \"ves_interval\": \"$ves_interval\", + \"ves_version\": \"$ves_version\", + \"ves_influxdb_host\": \"$ves_influxdb_host\", + \"ves_influxdb_port\": \"$ves_influxdb_port\", + \"ves_grafana_host\": \"$ves_grafana_host\", + \"ves_grafana_port\": \"$ves_grafana_port\", + \"ves_grafana_auth\": \"$ves_grafana_auth\", + \"ves_loglevel\": \"$ves_loglevel\"}" - if [[ "$master" == "$workers" ]]; then - nodes=$master + source ~/models/tools/cloudify/k8s-cloudify.sh start \ + ves-collector ves-collector "$inputs" + source ~/models/tools/cloudify/k8s-cloudify.sh clusterIp ves-collector + update_env ves_host $clusterIp + log "updated VES env"; env | grep ves else - nodes="$master $workers" - fi - - for node in $nodes; do - log "remove config for VES at node $node" ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - $k8s_user@$node <<EOF -sudo rm -rf /home/$k8s_user/ves -sudo rm -rf /home/$k8s_user/collectd + $k8s_user@$k8s_master <<EOF +sudo docker run -it -d -p 3001:3001 \ + -e ves_host=$ves_host \ + -e ves_port=$ves_port \ + -e ves_path=$ves_path \ + -e ves_topic=$ves_topic \ + -e ves_https=$ves_https \ + -e ves_user=$ves_user \ + -e ves_pass=$ves_pass \ + -e ves_interval=$ves_interval \ + -e ves_version=$ves_version \ + -e ves_influxdb_host=$ves_influxdb_host \ + -e ves_grafana_port=$ves_grafana_port \ + -e ves_grafana_host=$ves_grafana_host \ + -e ves_grafana_auth=$ves_grafana_auth \ + -e ves_loglevel=$ves_loglevel \ + --name ves-collector blsaws/ves-collector:latest EOF - done - - log "VES datasources and dashboards at grafana server, if needed" - curl -X DELETE \ - http://$ves_grafana_auth@$ves_grafana_host/api/datasources/name/VESEvents - curl -X DELETE \ - http://$ves_grafana_auth@$ves_grafana_host/api/dashboards/db/ves-demo + fi - log "Remove VES containers and collectd config at master node" - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - $k8s_user@$master <<'EOF' -cs="ves-agent ves-collector ves-grafana ves-influxdb ves-barometer" -for c in $cs; do - sudo docker stop $c - sudo docker rm -v $c -done -EOF + # debug hints + # curl 'http://172.16.0.5:30886/query?pretty=true&db=veseventsdb&q=SELECT%20moving_average%28%22load-shortterm%22%2C%205%29%20FROM%20%22load%22%20WHERE%20time%20%3E%3D%20now%28%29%20-%205m%20GROUP%20BY%20%22system%22' + # sudo docker logs ves-collector + # sudo docker exec -it ves-collector apt-get install -y tcpdump + # sudo docker exec -it ves-collector tcpdump -A -v -s 0 -i any port 3001 + # curl http://$ves_host:3001 + # sudo docker exec -it ves-collector /bin/bash } function verify_veseventsdb() { - source ~/k8s_env.sh + log "VES environment as set by ves_env.sh" + env | grep ves + for host in $1; do uuid=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$host sudo cat /sys/class/dmi/id/product_uuid) echo "$host=$uuid" - result=$(curl -G "http://$ves_influxdb_host/query?pretty=true" --data-urlencode "db=veseventsdb" --data-urlencode "q=SELECT moving_average(\"$3\", 5) FROM \"$2\" WHERE (\"system\" =~ /^($uuid)$/) AND time >= now() - 5m" | jq -r '.results[0].series') + result=$(curl -G "http://$ves_influxdb_host:$ves_influxdb_port/query?pretty=true" --data-urlencode "db=veseventsdb" --data-urlencode "q=SELECT moving_average(\"$3\", 5) FROM \"$2\" WHERE (\"system\" =~ /^($uuid)$/) AND time >= now() - 5m" | jq -r '.results[0].series') if [[ "$result" != "null" ]]; then echo "$host load data found in influxdb" else @@ -537,32 +425,41 @@ if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then sudo tee -a /etc/hosts fi +source ~/k8s_env.sh +if [[ -f ~/ves/tools/ves_env.sh ]]; then + source ~/ves/tools/ves_env.sh +fi +log "VES environment as input" +env | grep ves_ + +trap 'fail' ERR + case "$1" in - "collectd") - setup_collectd + "env") + setup_env + ;; + "barometer") + setup_barometer ;; "agent") setup_agent $2 ;; + "influxdb") + setup_influxdb + ;; + "grafana") + setup_grafana + ;; "collector") - setup_collector + setup_collector $2 ;; "kafka") - log "setup kafka server" - source ~/k8s_env.sh - sudo docker run -it -d -p 2181:2181 --name zookeeper zookeeper - sudo docker run -it -d -p 9092:9092 --name ves-kafka \ - -e zookeeper_host=$k8s_master_host \ - -e zookeeper=$k8s_master \ - -e kafka_hostname=$ves_kafka_hostname \ - blsaws/ves-kafka:latest + setup_kafka $2 ;; "verify") - verify_veseventsdb "$1" "load" "load-shortterm" - ;; - "clean") - clean $2 "$3" + verify_veseventsdb "$1" load load-shortterm ;; *) grep '#. ' $0 esac +trap '' ERR |