From da9564a9b0b78bbe341de9b039aab3c378eb027f Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Mon, 22 Jan 2018 15:25:53 -0800 Subject: Implement component deployment via cloudify JIRA: VES-2 Change-Id: Ic696f13d2a32e10663f50cd4e26b9a060525ff92 Signed-off-by: Bryan Sullivan --- tools/ves-setup.sh | 546 ++++++++++++++++++++++++----------------------------- 1 file changed, 246 insertions(+), 300 deletions(-) (limited to 'tools/ves-setup.sh') diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh index 0a279f0..1a3dbbe 100644 --- a/tools/ves-setup.sh +++ b/tools/ves-setup.sh @@ -35,7 +35,7 @@ #. - shell environment variables setup as below (for non-default setting) #. ves_mode: install mode (node|guest) for VES collectd plugin (default: node) #. ves_host: VES collector IP or hostname (default: 127.0.0.1) -#. ves_port: VES collector port (default: 30000) +#. ves_port: VES collector port (default: 3001) #. ves_path: REST path optionalRoutingPath element (default: empty) #. ves_topic: REST path topicName element (default: empty) #. ves_https: use HTTPS instead of HTTP (default: false) @@ -97,11 +97,13 @@ function common_prereqs() { } function setup_env() { + log "updating VES environment variables" cat <~/ves/tools/ves_env.sh #!/bin/bash ves_mode="${ves_mode:=node}" ves_host="${ves_host:=127.0.0.1}" -ves_port="${ves_port:=30000}" +ves_hostname="${ves_hostname:=ves-collector-service.default.svc.cluster.local}" +ves_port="${ves_port:=3001}" ves_path="${ves_path:=}" ves_topic="${ves_topic:=}" ves_https="${ves_https:=false}" @@ -109,17 +111,25 @@ ves_user="${ves_user:=}" ves_pass="${ves_pass:=}" ves_interval="${ves_interval:=20}" ves_version="${ves_version:=5.1}" +ves_zookeeper_hostname="${ves_zookeeper_hostname:=ves-zookeeper-service.default.svc.cluster.local}" +ves_zookeeper_host="${ves_zookeeper_host:=127.0.0.1}" +ves_zookeeper_port="${ves_zookeeper_port:=2181}" ves_kafka_host="${ves_kafka_host:=127.0.0.1}" -ves_kafka_hostname="${ves_kafka_hostname:=localhost}" +ves_kafka_hostname="${ves_kafka_hostname:=ves-kafka-service.default.svc.cluster.local}" ves_kafka_port="${ves_kafka_port:=9092}" -ves_influxdb_host="${ves_influxdb_host:=localhost:8086}" +ves_influxdb_host="${ves_influxdb_host:=127.0.0.1}" +ves_influxdb_hostname="${ves_influxdb_hostname:=ves-influxdb-service.default.svc.cluster.local}" +ves_influxdb_port="${ves_influxdb_port:=8086}" ves_influxdb_auth="${ves_influxdb_auth:=}" -ves_grafana_host="${ves_grafana_host:=localhost:3000}" +ves_grafana_host="${ves_grafana_host:=127.0.0.1}" +ves_grafana_hostname="${ves_grafana_hostname:=ves-grafana-service.default.svc.cluster.local}" +ves_grafana_port="${ves_grafana_port:=3000}" ves_grafana_auth="${ves_grafana_auth:=admin:admin}" -ves_loglevel="${ves_loglevel:=}" +ves_loglevel="${ves_loglevel:=DEBUG}" ves_cloudtype="${ves_cloudtype:=kubernetes}" export ves_mode export ves_host +export ves_hostname export ves_port export ves_path export ves_topic @@ -127,177 +137,105 @@ export ves_https export ves_user export ves_pass export ves_interval +export ves_version +export ves_zookeeper_host +export ves_zookeeper_hostname +export ves_zookeeper_port export ves_kafka_host export ves_kafka_hostname export ves_kafka_port export ves_influxdb_host +export ves_influxdb_hostname +export ves_influxdb_port export ves_influxdb_auth export ves_grafana_host +export ves_grafana_hostname +export ves_grafana_port export ves_grafana_auth export ves_loglevel export ves_cloudtype EOF source ~/ves/tools/ves_env.sh - echo ~/ves/tools/ves_env.sh + env | grep ves_ } -function setup_collectd() { - log "setup collectd" - - common_prereqs - source ~/ves/tools/ves_env.sh - log "VES environment as set by ves_env.sh" - env | grep ves +function update_env() { + log "update VES environment with $1=$2" + eval ${1}=${2} + export $1 + sed -i -- "s/.*$1=.*/$1=$2/" ~/ves/tools/ves_env.sh + env | grep ves_ +} - log "setup VES collectd config for VES $ves_mode mode" - mkdir ~/collectd - if [[ "$ves_mode" == "node" ]]; then -# # TODO: fix for journalctl -xe report "... is marked executable" -# sudo chmod 744 /etc/systemd/system/collectd.service - - cat <~/collectd/collectd.conf -# for VES plugin -LoadPlugin logfile - - LogLevel debug - File STDOUT - Timestamp true - PrintSeverity false - - -LoadPlugin csv - - DataDir "/work-dir/collectd/install/var/lib/csv" - StoreRates false - - -LoadPlugin target_set -LoadPlugin match_regex - - - - Plugin "^memory$" - - - PluginInstance "host" - - - - -LoadPlugin cpu - - ReportByCpu true - ReportByState true - ValuesPercentage true - - -LoadPlugin interface -LoadPlugin memory -LoadPlugin load -LoadPlugin disk -# TODO: how to set this option only to apply to VMs (not nodes) -LoadPlugin uuid - -LoadPlugin write_kafka - - Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port" - - Format JSON - - -EOF +function setup_kafka() { + log "setup kafka server" + log "deploy zookeeper and kafka" + if [[ "$1" == "cloudify" ]]; then + cp -r ~/ves/tools/cloudify/ves-zookeeper ~/models/tools/cloudify/blueprints/. + source ~/models/tools/cloudify/k8s-cloudify.sh start ves-zookeeper ves-zookeeper + source ~/models/tools/cloudify/k8s-cloudify.sh clusterIp ves-zookeeper + update_env ves_zookeeper_host $clusterIp - if [[ -d /etc/nova ]]; then - cat <>~/collectd/collectd.conf -LoadPlugin virt - - Connection "qemu:///system" - RefreshInterval 60 - HostnameFormat uuid - PluginInstanceFormat name - ExtraStats "cpu_util" - -EOF - fi + cp -r ~/ves/tools/cloudify/ves-kafka ~/models/tools/cloudify/blueprints/. + inputs="{ \ + \"zookeeper_hostname\": \"$ves_zookeeper_hostname\", + \"zookeeper_host\": \"$ves_zookeeper_host\", + \"zookeeper_port\": \"$ves_zookeeper_port\", + \"kafka_port\": \"$ves_kafka_port\", + \"kafka_hostname\": \"$ves_kafka_hostname\"}" + + source ~/models/tools/cloudify/k8s-cloudify.sh start ves-kafka ves-kafka "$inputs" + source ~/models/tools/cloudify/k8s-cloudify.sh clusterIp ves-kafka + update_env ves_kafka_host $clusterIp else - cat <~/collectd/collectd.conf -# for VES plugin -LoadPlugin logfile - - LogLevel debug - File STDOUT - Timestamp true - PrintSeverity false - - -LoadPlugin cpu - - ReportByCpu true - ReportByState true - ValuesPercentage true - - -LoadPlugin csv - - DataDir "/tmp" - - -LoadPlugin interface -LoadPlugin memory -LoadPlugin load -LoadPlugin disk -LoadPlugin uuid - -LoadPlugin write_kafka - - Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port" - - Format JSON - - - -LoadPlugin target_set -LoadPlugin match_regex - - - - Plugin "^memory$" - - - PluginInstance "guest" - - - + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master </$ves_mode/" \ + /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +sed -i -- "s//$ves_kafka_hostname/" \ + /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +sed -i -- "s//$ves_kafka_port/" \ + /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +kubectl create \ + -f /home/$k8s_user/ves/tools/kubernetes/ves-barometer/daemonset.yaml +EOF + +# sudo docker run -tid --net=host --name ves-barometer \ +# -v ~/collectd:/opt/collectd/etc/collectd.conf.d \ +# -v /var/run:/var/run -v /tmp:/tmp --privileged \ +# opnfv/barometer:latest /run_collectd.sh } function setup_agent() { log "setup VES agent" - source ~/k8s_env.sh - source ~/ves/tools/ves_env.sh - log "VES environment as set by ves_env.sh" - env | grep ves log "deploy the VES agent container" if [[ "$1" == "cloudify" ]]; then cp -r ~/ves/tools/cloudify/ves-agent ~/models/tools/cloudify/blueprints/. - # Cloudify is deployed on the k8s master node - manager_ip=$k8s_master - inputs="{ \ \"ves_mode\": \"$ves_mode\", \"ves_host\": \"$ves_host\", @@ -309,29 +247,32 @@ function setup_agent() { \"ves_pass\": \"$ves_pass\", \"ves_interval\": \"$ves_interval\", \"ves_version\": \"$ves_version\", - \"ves_kafka_port\": \"$ves_kafka_port\", - \"ves_kafka_host\": \"$ves_kafka_host\", \"ves_kafka_hostname\": \"$ves_kafka_hostname\", + \"ves_kafka_host\": \"$ves_kafka_host\", + \"ves_kafka_port\": \"$ves_kafka_port\", \"ves_loglevel\": \"$ves_loglevel\"}" bash ~/models/tools/cloudify/k8s-cloudify.sh start ves-agent ves-agent "$inputs" else - sudo docker run -it -d \ - -e ves_mode=$ves_mode \ - -e ves_host=$ves_host \ - -e ves_port=$ves_port \ - -e ves_path=$ves_path \ - -e ves_topic=$ves_topic \ - -e ves_https=$ves_https \ - -e ves_user=$ves_user \ - -e ves_pass=$ves_pass \ - -e ves_interval=$ves_interval \ - -e ves_version=$ves_version \ - -e ves_kafka_port=$ves_kafka_port \ - -e ves_kafka_host=$ves_kafka_host \ - -e ves_kafka_hostname=$ves_kafka_hostname \ - -e ves_loglevel=$ves_loglevel \ - --name ves-agent blsaws/ves-agent:latest + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <~/ves/tools/grafana/datasource.json -{ "name":"VESEvents", - "type":"influxdb", - "access":"direct", - "url":"http://$ves_host:8086", - "password":"root", - "user":"root", - "database":"veseventsdb", - "basicAuth":false, - "basicAuthUser":"", - "basicAuthPassword":"", - "withCredentials":false, - "isDefault":false, - "jsonData":null -} + log "checking for grafana at http://$ves_grafana_host:$ves_grafana_port" + if ! curl http://$ves_grafana_host:$ves_grafana_port ; then + log "install Grafana container on k8s master" + update_env ves_grafana_host $k8s_master + update_env ves_grafana_hostname $k8s_master_hostname + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master \ + sudo docker run -d --name ves-grafana -p 3000:3000 grafana/grafana + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <<'EOF' +status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') +while [[ "x$status" != "xrunning" ]]; do + echo; echo "Grafana container state is ($status)" + sleep 10 +status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status') +done +echo; echo "Grafana container state is $status" EOF - - # Use /home/$USER/ instead of ~ with @ - curl -H "Accept: application/json" -H "Content-type: application/json" \ - -X POST -d @/home/$USER/ves/tools/grafana/datasource.json \ - http://$ves_grafana_auth@$ves_grafana_host/api/datasources - - log "add VES dashboard to Grafana at http://$ves_grafana_auth@$ves_grafana_host" - curl -H "Accept: application/json" -H "Content-type: application/json" \ - -X POST \ - -d @/home/$USER/ves/tools/grafana/Dashboard.json\ - http://$ves_grafana_auth@$ves_grafana_host/api/dashboards/db + fi log "setup collector container" - # TODO: migrate to deployment via Helm - sudo docker run -it -d -p 30000:30000 \ - -e ves_host=$ves_host \ - -e ves_port=$ves_port \ - -e ves_path=$ves_path \ - -e ves_topic=$ves_topic \ - -e ves_https=$ves_https \ - -e ves_user=$ves_user \ - -e ves_pass=$ves_pass \ - -e ves_interval=$ves_interval \ - -e ves_version=$ves_version \ - -e ves_influxdb_host=$ves_influxdb_host \ - -e ves_loglevel=$ves_loglevel \ - --name ves-collector blsaws/ves-collector:latest + if [[ "$1" == "cloudify" ]]; then + cp -r ~/ves/tools/cloudify/ves-collector ~/models/tools/cloudify/blueprints/. + inputs="{ \ + \"ves_host\": \"$ves_host\", + \"ves_port\": \"$ves_port\", + \"ves_path\": \"$ves_path\", + \"ves_topic\": \"$ves_topic\", + \"ves_https\": \"$ves_https\", + \"ves_user\": \"$ves_user\", + \"ves_pass\": \"$ves_pass\", + \"ves_interval\": \"$ves_interval\", + \"ves_version\": \"$ves_version\", + \"ves_influxdb_host\": \"$ves_influxdb_host\", + \"ves_influxdb_port\": \"$ves_influxdb_port\", + \"ves_grafana_host\": \"$ves_grafana_host\", + \"ves_grafana_port\": \"$ves_grafana_port\", + \"ves_grafana_auth\": \"$ves_grafana_auth\", + \"ves_loglevel\": \"$ves_loglevel\"}" + + source ~/models/tools/cloudify/k8s-cloudify.sh start \ + ves-collector ves-collector "$inputs" + source ~/models/tools/cloudify/k8s-cloudify.sh clusterIp ves-collector + update_env ves_host $clusterIp + log "updated VES env"; env | grep ves + else + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $k8s_user@$k8s_master <= now() - 5m" | jq -r '.results[0].series') + result=$(curl -G "http://$ves_influxdb_host:$ves_influxdb_port/query?pretty=true" --data-urlencode "db=veseventsdb" --data-urlencode "q=SELECT moving_average(\"$3\", 5) FROM \"$2\" WHERE (\"system\" =~ /^($uuid)$/) AND time >= now() - 5m" | jq -r '.results[0].series') if [[ "$result" != "null" ]]; then echo "$host load data found in influxdb" else @@ -527,8 +476,14 @@ if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then sudo tee -a /etc/hosts fi +source ~/k8s_env.sh +if [[ -f ~/ves/tools/ves_env.sh ]]; then + source ~/ves/tools/ves_env.sh +fi log "VES environment as input" -env | grep ves +env | grep ves_ + +trap 'fail' ERR case "$1" in "env") @@ -541,23 +496,13 @@ case "$1" in setup_agent $2 ;; "collector") - setup_collector + setup_collector $2 ;; "kafka") - log "setup kafka server" - source ~/k8s_env.sh - source ~/ves/tools/ves_env.sh - log "VES environment as set by ves_env.sh" - env | grep ves - sudo docker run -it -d -p 2181:2181 --name ves-zookeeper zookeeper - sudo docker run -it -d -p 9092:9092 --name ves-kafka \ - -e zookeeper_host=$k8s_master_host \ - -e zookeeper=$k8s_master \ - -e kafka_hostname=$ves_kafka_hostname \ - blsaws/ves-kafka:latest + setup_kafka $2 ;; "verify") - verify_veseventsdb "$1" "load" "load-shortterm" + verify_veseventsdb "$1" load load-shortterm ;; "clean") clean $2 "$3" $4 @@ -565,3 +510,4 @@ case "$1" in *) grep '#. ' $0 esac +trap '' ERR -- cgit 1.2.3-korg