diff options
Diffstat (limited to 'tools/prometheus/prometheus-tools.sh')
| -rw-r--r-- | tools/prometheus/prometheus-tools.sh | 209 |
1 files changed, 156 insertions, 53 deletions
diff --git a/tools/prometheus/prometheus-tools.sh b/tools/prometheus/prometheus-tools.sh index 533b84c..8463231 100644 --- a/tools/prometheus/prometheus-tools.sh +++ b/tools/prometheus/prometheus-tools.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2017 AT&T Intellectual Property, Inc +# Copyright 2017-2018 AT&T Intellectual Property, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,11 +19,22 @@ #. Prerequisites: #. - Ubuntu server for master and agent nodes #. - Docker installed +#. - For helm-based install, k8s+helm installed #. Usage: #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models #. $ cd ~/models/tools/prometheus -#. $ bash prometheus-tools.sh setup -#. $ bash prometheus-tools.sh clean +#. $ bash prometheus-tools.sh <setup|clean> prometheus <docker|helm> <"agents"> +#. prometheus: setup/clean prometheus +#. docker: setup/clean via docker +#. helm: setup/clean via helm +#. agents: for docker-based setup, a quoted, space-separated list agent nodes +#. note: node running this script must have ssh-key enabled access to agents +#. $ bash prometheus-tools.sh <setup|clean> grafana <docker|helm> [URI] [creds] +#. grafana: setup/clean grafana +#. docker: setup/clean via docker +#. helm: setup/clean via helm +#. URI: optional URI of grafana server to use +#. creds: optional grafana credentials (default: admin:admin) # # Prometheus links @@ -49,21 +60,55 @@ function fail() { } function setup_prometheus() { - # Prerequisites - sudo apt install -y golang-go jq - host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') - - # Install Prometheus server - # TODO: add --set server.persistentVolume.storageClass=general - # TODO: add persistent volume support - log "Setup prometheus via Helm" - helm install stable/prometheus --name pm \ - --set alertmanager.enabled=false \ - --set pushgateway.enabled=false \ - --set server.service.nodePort=30990 \ - --set server.service.type=NodePort \ - --set server.persistentVolume.enabled=false + log "Setup prometheus" + log "Setup prerequisites" + if [[ "$dist" == "ubuntu" ]]; then + sudo apt-get install -y golang-go jq + else + sudo yum install -y golang-go jq + fi + if [[ "$how" == "docker" ]]; then + log "Deploy prometheus node exporter on each agent node" + for agent in $agents ; do + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $dist@$agent sudo docker run -d --restart=always -p 9101:9101 \ + -p 9100:9100 --name prometheus-node-exporter prom/node-exporter + done + log "Create prometheus config file prometheus.yml" + cat <<'EOF' >prometheus.yml +global: + scrape_interval: 15s +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: +EOF + for agent in $agents; do + echo " - targets: ['${agent}:9100']" >>prometheus.yml + echo " - targets: ['${agent}:9101']" >>prometheus.yml + done + log "prometheus.yaml:" + cat prometheus.yml + log "Start prometheus server" + sudo docker run -d --restart=unless-stopped -p 9090:9090 -p 30990:9090 \ + -v /home/$USER/prometheus.yml:/etc/prometheus/prometheus.yml \ + --name prometheus prom/prometheus + fi + if [[ "$how" == "helm" ]]; then + # Install Prometheus server + # TODO: add --set server.persistentVolume.storageClass=general + # TODO: add persistent volume support + log "Setup prometheus server and agents via Helm" + helm install stable/prometheus --name pm \ + --set alertmanager.enabled=false \ + --set pushgateway.enabled=false \ + --set server.service.nodePort=30990 \ + --set server.service.type=NodePort \ + --set server.persistentVolume.enabled=false + fi + + host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') while ! curl -o ~/tmp/up http://$host_ip:30990/api/v1/query?query=up ; do log "Prometheus API is not yet responding... waiting 10 seconds" sleep 10 @@ -78,26 +123,35 @@ function setup_prometheus() { log "$job at $eip" done log "Prometheus dashboard is available at http://$host_ip:30990" - echo "Prometheus dashboard is available at http://$host_ip:30990" >>~/tmp/summary } function setup_grafana() { - # TODO: use randomly generated password - # TODO: add persistent volume support - log "Setup grafana via Helm" - #TODSO: add --set server.persistentVolume.storageClass=general - helm install --name gf stable/grafana \ - --set server.service.nodePort=30330 \ - --set server.service.type=NodePort \ - --set server.adminPassword=admin \ - --set server.persistentVolume.enabled=false + host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + if [[ "$grafana" == "" ]]; then + if [[ "$how" == "docker" ]]; then + log "Setup grafana via docker" + docker run -d --name=grafana -p 30330:3000 grafana/grafana + fi + if [[ "$how" == "helm" ]]; then + # TODO: use randomly generated password + # TODO: add persistent volume support + log "Setup grafana via Helm" + #TODO: add --set server.persistentVolume.storageClass=general + helm install --name gf stable/grafana \ + --set server.service.nodePort=30330 \ + --set server.service.type=NodePort \ + --set server.adminPassword=admin \ + --set server.persistentVolume.enabled=false + fi + grafana=$host_ip:30330 + fi log "Setup Grafana datasources and dashboards" - host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') prometheus_ip=$host_ip - grafana_ip=$host_ip - - while ! curl -X POST http://admin:admin@$grafana_ip:30330/api/login/ping ; do + if [[ "$creds" == "" ]]; then + creds="admin:admin" + fi + while ! curl -X POST http://$creds@$grafana/api/login/ping ; do log "Grafana API is not yet responding... waiting 10 seconds" sleep 10 done @@ -110,7 +164,7 @@ function setup_grafana() { EOF curl -X POST -o ~/tmp/json -u admin:admin -H "Accept: application/json" \ -H "Content-type: application/json" \ - -d @datasources.json http://admin:admin@$grafana_ip:30330/api/datasources + -d @datasources.json http://$creds@$grafana/api/datasources if [[ "$(jq -r '.message' ~/tmp/json)" != "Datasource added" ]]; then fail "Datasource creation failed" @@ -126,39 +180,88 @@ EOF cd $WORK_DIR/dashboards boards=$(ls) for board in $boards; do - curl -X POST -u admin:admin -H "Accept: application/json" -H "Content-type: application/json" -d @${board} http://$grafana_ip:30330/api/dashboards/db + curl -X POST -u admin:admin \ + -H "Accept: application/json" -H "Content-type: application/json" \ + -d @${board} http://$creds@$grafana/api/dashboards/db done log "Grafana dashboards are available at http://$host_ip:30330 (login as admin/admin)" - echo "Grafana dashboards are available at http://$host_ip:30330 (login as admin/admin)" >>~/tmp/summary log "Grafana API is available at http://admin:admin@$host_ip:30330/api/v1/query?query=<string>" - echo "Grafana API is available at http://admin:admin@$host_ip:30330/api/v1/query?query=<string>" >>~/tmp/summary log "connect_grafana complete" } +function clean_prometheus() { + if [[ "$how" == "docker" ]]; then + log "Clean prometheus via docker" + sudo docker stop prometheus + sudo docker rm -v prometheus + for agent in $agents ; do + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $dist@$agent sudo docker stop prometheus-node-exporter + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $dist@$agent sudo docker rm -v prometheus-node-exporter + done + fi + if [[ "$how" == "helm" ]]; then + log "Clean prometheus via Helm" + helm delete --purge pm + fi +} + +function clean_grafana() { + if [[ "$grafana" == "" ]]; then + log "Delete grafana server" + host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + grafana=$host_ip:30330 + if [[ "$how" == "docker" ]]; then + sudo docker stop grafana + sudo docker rm grafana + fi + if [[ "$how" == "helm" ]]; then + helm delete gf + fi + else + if [[ "$creds" == "" ]]; then + creds="admin:admin" + fi + log "Delete prometheus datasource at grafana server" + curl -X DELETE http://$creds@$grafana/api/datasources/name/Prometheus + log "Delete prometheus dashboards at grafana server" + boards="docker-dashboard docker-host-and-container-overview node-exporter-server-metrics node-exporter-single-server" + for board in $boards; do + curl -X DELETE http://$creds@$grafana/api/dashboards/db/$board + done + fi +} + export WORK_DIR=$(pwd) +dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}') + +what=$2 +how=$3 case "$1" in setup) - setup_prometheus - setup_grafana + if [[ "$what" == "prometheus" ]]; then + agents="$4" + setup_prometheus + fi + if [[ "$what" == "grafana" ]]; then + grafana="$4" + creds="$5" + setup_grafana + fi ;; clean) - sudo kill $(ps -ef | grep "\./prometheus" | grep prometheus.yml | awk '{print $2}') - rm -rf ~/prometheus - sudo docker stop grafana - sudo docker rm grafana - for node in $nodes; do - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$node "sudo kill $(ps -ef | grep ./node_exporter | awk '{print $2}')" - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$node "rm -rf /home/ubuntu/node_exporter" - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$node "sudo kill $(ps -ef | grep ./haproxy_exporter | awk '{print $2}')" - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$node "rm -rf /home/ubuntu/haproxy_exporter" - done + if [[ "$what" == "prometheus" ]]; then + agents="$4" + clean_prometheus + fi + if [[ "$what" == "grafana" ]]; then + grafana="$4" + creds="$5" + clean_grafana + fi ;; *) grep '#. ' $0 -esac -cat ~/tmp/summary +esac
\ No newline at end of file |