From c02beb3505facd5ab514b9786203d6f73d6a5227 Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Wed, 10 Jan 2018 08:07:30 -0800 Subject: Use Helm chart to deploy Prometheus and Grafana JIRA: MODELS-2 Change-Id: Ia7f39251a941b576df9f11791aad6f71f074fa0e Signed-off-by: Bryan Sullivan --- tools/kubernetes/demo_deploy.sh | 10 +-- tools/kubernetes/k8s-cluster.sh | 4 +- tools/prometheus/prometheus-tools.sh | 151 ++++++++++------------------------- 3 files changed, 49 insertions(+), 116 deletions(-) diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh index fa347af..2ec3b5f 100644 --- a/tools/kubernetes/demo_deploy.sh +++ b/tools/kubernetes/demo_deploy.sh @@ -123,8 +123,8 @@ if [[ "$k8s_master" != "$k8s_workers" ]]; then echo; echo "$0 $(date): Setting up kubernetes workers..." run_master "bash k8s-cluster.sh workers \"$k8s_workers\"" else - echo; echo "Enable pod scheduling on master node" - run_master "kubectl taint node $k8s_nodes node-role.kubernetes.io/master:NoSchedule-" + echo; echo "Label $k8s_master_host for role=worker" + run_master "kubectl label nodes $k8s_master_host role=worker --overwrite" fi echo; echo "$0 $(date): Setting up helm..." @@ -147,7 +147,7 @@ fi echo; echo "Setting up Prometheus..." scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \ $k8s_user@$k8s_master:/home/$k8s_user/. -run_master "bash prometheus-tools.sh all \"$k8s_workers\"" +run_master "bash prometheus-tools.sh setup" echo; echo "$0 $(date): Setting up cloudify..." scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \ @@ -166,7 +166,7 @@ if [[ ! -d ~/ves ]]; then fi ves_influxdb_host=$k8s_master:8086 export ves_influxdb_host -ves_grafana_host=$k8s_master:3000 +ves_grafana_host=$k8s_master:30330 export ves_grafana_host ves_grafana_auth=admin:admin export ves_grafana_auth @@ -185,7 +185,7 @@ runtime=$((deploy_end-deploy_start)) log "Deploy \"$1\" duration = $runtime minutes" port=$(bash ~/models/tools/cloudify/k8s-cloudify.sh port nginx) -echo "Prometheus UI is available at http://$k8s_master:9090" +echo "Prometheus UI is available at http://$k8s_master:30990" echo "InfluxDB API is available at http://$ves_influxdb_host/query&db=veseventsdb&q=" echo "Grafana dashboards are available at http://$ves_grafana_host (login as $ves_grafana_auth)" echo "Grafana API is available at http://$ves_grafana_auth@$ves_grafana_host/api/v1/query?query=" diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index aa3f28f..65f19b5 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -46,6 +46,8 @@ #. Status: work in progress, incomplete # +trap 'fail' ERR + function fail() { log $1 exit 1 @@ -227,7 +229,7 @@ function setup_k8s_master() { log "Allow pod scheduling on master (nodeSelector will be used to limit them)" kubectl taint node $HOSTNAME node-role.kubernetes.io/master:NoSchedule- - log "Label node $HOSTNAME as 'master'" + log "Label node $HOSTNAME as 'role=master'" kubectl label nodes $HOSTNAME role=master } diff --git a/tools/prometheus/prometheus-tools.sh b/tools/prometheus/prometheus-tools.sh index 04a2623..12650c8 100644 --- a/tools/prometheus/prometheus-tools.sh +++ b/tools/prometheus/prometheus-tools.sh @@ -22,13 +22,8 @@ #. Usage: #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models #. $ cd ~/models/tools/prometheus -#. $ bash prometheus-tools.sh setup "" -#. : space separated IP of agent nodes -#. $ bash prometheus-tools.sh grafana -#. Runs grafana in a docker container and connects to prometheus as datasource -#. $ bash prometheus-tools.sh all "" -#. Does all of the above -#. $ bash prometheus-tools.sh clean "" +#. $ bash prometheus-tools.sh setup +#. $ bash prometheus-tools.sh clean # # Prometheus links @@ -55,74 +50,21 @@ function fail() { function setup_prometheus() { # Prerequisites - log "Setting up prometheus master and agents" sudo apt install -y golang-go jq + host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') # Install Prometheus server - log "Setting up prometheus master" - if [[ -d ~/prometheus ]]; then rm -rf ~/prometheus; fi - mkdir ~/prometheus - cd ~/prometheus - wget https://github.com/prometheus/prometheus/releases/download/v2.0.0-beta.2/prometheus-2.0.0-beta.2.linux-amd64.tar.gz - tar xvfz prometheus-*.tar.gz - cd prometheus-* - # Customize prometheus.yml below for your server IPs - # This example assumes the node_exporter and haproxy_exporter will be installed on each node - cat <<'EOF' >prometheus.yml -global: - scrape_interval: 15s # By default, scrape targets every 15 seconds. - - # Attach these labels to any time series or alerts when communicating with - # external systems (federation, remote storage, Alertmanager). - external_labels: - monitor: 'codelab-monitor' - -# A scrape configuration containing exactly one endpoint to scrape: -# Here it's Prometheus itself. -scrape_configs: - # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'prometheus' - - # Override the global default and scrape targets from this job every 5 seconds. - scrape_interval: 5s - - static_configs: -EOF - - for node in $nodes; do - echo " - targets: ['${node}:9100']" >>prometheus.yml - echo " - targets: ['${node}:9101']" >>prometheus.yml - done - - # Start Prometheus - nohup ./prometheus --config.file=prometheus.yml > /dev/null 2>&1 & - # Browse to http://host_ip:9090 - - log "Installing exporters" - # Install exporters - # https://github.com/prometheus/node_exporter - cd ~/prometheus - wget https://github.com/prometheus/node_exporter/releases/download/v0.14.0/node_exporter-0.14.0.linux-amd64.tar.gz - tar xvfz node*.tar.gz - # https://github.com/prometheus/haproxy_exporter - wget https://github.com/prometheus/haproxy_exporter/releases/download/v0.7.1/haproxy_exporter-0.7.1.linux-amd64.tar.gz - tar xvfz haproxy*.tar.gz - - # The scp and ssh actions below assume you have key-based access enabled to the nodes - for node in $nodes; do - log "Setup agent at $node" - scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - node_exporter-0.14.0.linux-amd64/node_exporter ubuntu@$node:/home/ubuntu/node_exporter - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$node "nohup ./node_exporter > /dev/null 2>&1 &" - scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - haproxy_exporter-0.7.1.linux-amd64/haproxy_exporter ubuntu@$node:/home/ubuntu/haproxy_exporter - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$node "nohup ./haproxy_exporter > /dev/null 2>&1 &" - done - - host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') - while ! curl -o /tmp/up http://$host_ip:9090/api/v1/query?query=up ; do + # TODO: add --set server.persistentVolume.storageClass=general + # TODO: add persistent volume support + log "Setup prometheus via Helm" + helm install stable/prometheus --name pm \ + --set alertmanager.enabled=false \ + --set pushgateway.enabled=false \ + --set server.service.nodePort=30990 \ + --set server.service.type=NodePort \ + --set server.persistentVolume.enabled=false + + while ! curl -o /tmp/up http://$host_ip:30990/api/v1/query?query=up ; do log "Prometheus API is not yet responding... waiting 10 seconds" sleep 10 done @@ -135,30 +77,40 @@ EOF job=$(jq -r ".data.result[$exp].metric.job" /tmp/up) log "$job at $eip" done - log "Prometheus dashboard is available at http://$host_ip:9090" - echo "Prometheus dashboard is available at http://$host_ip:9090" >>/tmp/summary + log "Prometheus dashboard is available at http://$host_ip:30990" + echo "Prometheus dashboard is available at http://$host_ip:30990" >>/tmp/summary } -function connect_grafana() { +function setup_grafana() { + # TODO: use randomly generated password + # TODO: add persistent volume support + log "Setup grafana via Helm" + #TODSO: add --set server.persistentVolume.storageClass=general + helm install --name gf stable/grafana \ + --set server.service.nodePort=30330 \ + --set server.service.type=NodePort \ + --set server.adminPassword=admin \ + --set server.persistentVolume.enabled=false + log "Setup Grafana datasources and dashboards" - prometheus_ip=$1 - grafana_ip=$2 + host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + prometheus_ip=$host_ip + grafana_ip=$host_ip - while ! curl -X POST http://admin:admin@$grafana_ip:3000/api/login/ping ; do + while ! curl -X POST http://admin:admin@$grafana_ip:30330/api/login/ping ; do log "Grafana API is not yet responding... waiting 10 seconds" sleep 10 done log "Setup Prometheus datasource for Grafana" - cd ~/prometheus/ cat >datasources.json <>/tmp/summary - log "Grafana API is available at http://admin:admin@$host_ip:3000/api/v1/query?query=" - echo "Grafana API is available at http://admin:admin@$host_ip:3000/api/v1/query?query=" >>/tmp/summary -} - -function run_and_connect_grafana() { - # Per http://docs.grafana.org/installation/docker/ - host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') - sudo docker run -d -p 3000:3000 --name grafana grafana/grafana - status=$(sudo docker inspect grafana | jq -r '.[0].State.Status') - while [[ "x$status" != "xrunning" ]]; do - log "Grafana container state is ($status)" - sleep 10 - status=$(sudo docker inspect grafana | jq -r '.[0].State.Status') - done - log "Grafana container state is $status" - - connect_grafana $host_ip $host_ip + log "Grafana dashboards are available at http://$host_ip:30330 (login as admin/admin)" + echo "Grafana dashboards are available at http://$host_ip:30330 (login as admin/admin)" >>/tmp/summary + log "Grafana API is available at http://admin:admin@$host_ip:30330/api/v1/query?query=" + echo "Grafana API is available at http://admin:admin@$host_ip:30330/api/v1/query?query=" >>/tmp/summary log "connect_grafana complete" } export WORK_DIR=$(pwd) -nodes=$2 + case "$1" in setup) - setup_prometheus "$2" - ;; - grafana) - run_and_connect_grafana - ;; - all) - setup_prometheus "$2" - run_and_connect_grafana + setup_prometheus + setup_grafana ;; clean) sudo kill $(ps -ef | grep "\./prometheus" | grep prometheus.yml | awk '{print $2}') -- cgit 1.2.3-korg