From da9564a9b0b78bbe341de9b039aab3c378eb027f Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Mon, 22 Jan 2018 15:25:53 -0800 Subject: Implement component deployment via cloudify JIRA: VES-2 Change-Id: Ic696f13d2a32e10663f50cd4e26b9a060525ff92 Signed-off-by: Bryan Sullivan --- build/ves-collector/Dashboard.json | 996 +++++++++++++++++++++++++++++++++++++ build/ves-collector/Dockerfile | 14 +- build/ves-collector/start.sh | 59 ++- 3 files changed, 1055 insertions(+), 14 deletions(-) create mode 100644 build/ves-collector/Dashboard.json (limited to 'build/ves-collector') diff --git a/build/ves-collector/Dashboard.json b/build/ves-collector/Dashboard.json new file mode 100644 index 0000000..b88646c --- /dev/null +++ b/build/ves-collector/Dashboard.json @@ -0,0 +1,996 @@ +{ +"dashboard": { + "description": "This Dashboard provides a general overview of a host, with templating to select the hostname.", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": 401, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 3, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + } + ], + "measurement": "load", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"cpusystem\") FROM \"cpu\" WHERE $timeFilter GROUP BY time(1m) fill(null)", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load-shortterm" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "host load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Percent", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 6, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "cpu" + ], + "type": "tag" + } + ], + "measurement": "cpuUsage", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"cpusystem\") FROM \"cpu\" WHERE $timeFilter GROUP BY time(1m) fill(null)", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "cpuUsageUser" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "host CPU Usage User", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Percent", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 442, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 2, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "vnic" + ], + "type": "tag" + } + ], + "measurement": "vNicPerformance", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT derivative(mean(\"rxoctetsacc\"), 10s) FROM \"vnic\" WHERE \"system\" = 'computehost' AND $timeFilter GROUP BY time(1m) fill(null)", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "receivedTotalPacketsAccumulated" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Received Octets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Octets/Packets", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 4, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "vnic" + ], + "type": "tag" + } + ], + "measurement": "vNicPerformance", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "receivedOctetsAccumulated" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmitted Octets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Octets/Packets", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 362, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "disk" + ], + "type": "tag" + } + ], + "measurement": "diskUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "diskOpsWriteLast" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + }, + { + "condition": "AND", + "key": "disk", + "operator": "=", + "value": "sda" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Usage SDA", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "disk" + ], + "type": "tag" + } + ], + "measurement": "diskUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "diskOpsWriteLast" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + }, + { + "condition": "AND", + "key": "disk", + "operator": "=", + "value": "sdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Usage SDB", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + } + ], + "measurement": "memoryUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "memoryUsed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "VESEvents", + "hide": 0, + "includeAll": true, + "label": "host", + "multi": true, + "name": "host", + "options": [], + "query": "SHOW TAG VALUES WITH KEY=system", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "10s", + "20s", + "30s", + "1m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "VES Demo", + "version": 4 +} +} diff --git a/build/ves-collector/Dockerfile b/build/ves-collector/Dockerfile index 9161871..4cd135f 100644 --- a/build/ves-collector/Dockerfile +++ b/build/ves-collector/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2017 AT&T Intellectual Property, Inc +# Copyright 2017-2018 AT&T Intellectual Property, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# What this is: A Dockerfile for building an OPFNV VES Agent container image. +# What this is: A Dockerfile for building an OPFNV VES Collector container image. # # Status: this is a work in progress, under test. # @@ -21,14 +21,9 @@ FROM ubuntu:xenial MAINTAINER Bryan Sullivan -RUN apt-get update -RUN apt-get install -y apt-utils +RUN apt-get update && apt-get install -y apt-utils RUN apt-get -y upgrade -RUN apt-get update -RUN apt-get install -y git - -# Required for VES collector -RUN apt-get install -y python-pip python-jsonschema +RUN apt-get update && apt-get install -y git python-pip python-jsonschema curl RUN pip install requests RUN mkdir /opt/ves @@ -37,5 +32,6 @@ RUN mkdir /opt/ves RUN git clone https://github.com/att/evel-test-collector.git /opt/ves/evel-test-collector COPY monitor.py /opt/ves/evel-test-collector/code/collector/monitor.py +COPY Dashboard.json /opt/ves/Dashboard.json COPY start.sh /opt/ves/start.sh ENTRYPOINT ["/bin/bash", "/opt/ves/start.sh"] diff --git a/build/ves-collector/start.sh b/build/ves-collector/start.sh index be30c9a..250af34 100644 --- a/build/ves-collector/start.sh +++ b/build/ves-collector/start.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2017 AT&T Intellectual Property, Inc +# Copyright 2017-2018 AT&T Intellectual Property, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,8 @@ sed -i -- \ evel-test-collector/config/collector.conf sed -i -- "s/vel_domain = 127.0.0.1/vel_domain = $ves_host/g" \ evel-test-collector/config/collector.conf +sed -i -- "s/vel_port = 30000/vel_port = $ves_port/g" \ + evel-test-collector/config/collector.conf sed -i -- "s/vel_username =/vel_username = $ves_user/g" \ evel-test-collector/config/collector.conf sed -i -- "s/vel_password =/vel_password = $ves_pass/g" \ @@ -31,18 +33,65 @@ sed -i -- "s~vel_path = vendor_event_listener/~vel_path = $ves_path~g" \ evel-test-collector/config/collector.conf sed -i -- "s~vel_topic_name = example_vnf~vel_topic_name = $ves_topic~g" \ evel-test-collector/config/collector.conf -sed -i -- "/vel_topic_name = /a influxdb = $ves_influxdb_host" \ +sed -i -- "/vel_topic_name = /a influxdb = $ves_influxdb_host:$ves_influxdb_port" \ evel-test-collector/config/collector.conf +echo; echo "evel-test-collector/config/collector.conf" +cat evel-test-collector/config/collector.conf + +echo; echo "wait for InfluxDB API at $ves_influxdb_host:$ves_influxdb_port" +while ! curl http://$ves_influxdb_host:$ves_influxdb_port/ping ; do + echo "InfluxDB API is not yet responding... waiting 10 seconds" + sleep 10 +done + +echo; echo "setup veseventsdb in InfluxDB" +# TODO: check if pre-existing and skip +curl -X POST http://$ves_influxdb_host:$ves_influxdb_port/query \ + --data-urlencode "q=CREATE DATABASE veseventsdb" + +echo; echo "wait for Grafana API to be active" +while ! curl http://$ves_grafana_host:$ves_grafana_port ; do + echo "Grafana API is not yet responding... waiting 10 seconds" + sleep 10 +done + +echo; echo "add VESEvents datasource to Grafana" +# TODO: check if pre-existing and skip +cat </opt/ves/datasource.json +{ "name":"VESEvents", + "type":"influxdb", + "access":"direct", + "url":"http://$ves_influxdb_host:$ves_influxdb_port", + "password":"root", + "user":"root", + "database":"veseventsdb", + "basicAuth":false, + "basicAuthUser":"", + "basicAuthPassword":"", + "withCredentials":false, + "isDefault":false, + "jsonData":null +} +EOF + +curl -H "Accept: application/json" -H "Content-type: application/json" \ + -X POST -d @/opt/ves/datasource.json \ + http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/datasources + +echo; echo "add VES dashboard to Grafana" +curl -H "Accept: application/json" -H "Content-type: application/json" \ + -X POST -d @/opt/ves/Dashboard.json \ + http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/dashboards/db + if [[ "$ves_loglevel" != "" ]]; then python /opt/ves/evel-test-collector/code/collector/monitor.py \ --config /opt/ves/evel-test-collector/config/collector.conf \ - --influxdb $ves_influxdb_host \ + --influxdb $ves_influxdb_host:$ves_influxdb_port \ --section default > /opt/ves/monitor.log 2>&1 else python /opt/ves/evel-test-collector/code/collector/monitor.py \ --config /opt/ves/evel-test-collector/config/collector.conf \ - --influxdb $ves_influxdb_host \ + --influxdb $ves_influxdb_host:$ves_influxdb_port \ --section default fi - -- cgit 1.2.3-korg