From a30888b3403bfd22358c30d9b706e73545fa5031 Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Tue, 21 Nov 2017 18:55:32 -0800 Subject: Add more collectd data to influx/grafana JIRA: VES-2 Disable uuid for "node" mode Change-Id: I055db4f52eee7a61438a4e7e63740b42dc3f1777 Signed-off-by: Bryan Sullivan --- tools/grafana/Dashboard.json | 380 ++++++++++++++++++++++++++++++++++++------- tools/monitor.py | 54 ++++++ tools/ves-setup.sh | 6 +- 3 files changed, 381 insertions(+), 59 deletions(-) diff --git a/tools/grafana/Dashboard.json b/tools/grafana/Dashboard.json index 3160296..2ea9eec 100644 --- a/tools/grafana/Dashboard.json +++ b/tools/grafana/Dashboard.json @@ -11,7 +11,7 @@ "rows": [ { "collapse": false, - "height": 273, + "height": 401, "panels": [ { "aliasColors": {}, @@ -31,13 +31,15 @@ "id": 3, "interval": "30s", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -79,6 +81,12 @@ "load-shortterm" ], "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" } ] ], @@ -140,15 +148,28 @@ "dashes": false, "datasource": "VESEvents", "fill": 1, - "id": 5, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 6, + "interval": "30s", "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -173,20 +194,33 @@ "system" ], "type": "tag" + }, + { + "params": [ + "cpu" + ], + "type": "tag" } ], - "measurement": "memoryUsage", + "measurement": "cpuUsage", "orderByTime": "ASC", - "policy": "autogen", - "refId": "A", + "policy": "default", + "query": "SELECT mean(\"cpusystem\") FROM \"cpu\" WHERE $timeFilter GROUP BY time(1m) fill(null)", + "refId": "B", "resultFormat": "time_series", "select": [ [ { "params": [ - "memoryFree" + "cpuUsageUser" ], "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" } ] ], @@ -202,13 +236,14 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Memory", + "title": "host CPU Usage User", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", + "x-axis": true, "xaxis": { "buckets": null, "mode": "time", @@ -216,11 +251,16 @@ "show": true, "values": [] }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], "yaxes": [ { "format": "short", - "label": null, - "logBase": 10, + "label": "Percent", + "logBase": 1, "max": null, "min": null, "show": true @@ -245,7 +285,7 @@ }, { "collapse": false, - "height": 433, + "height": 442, "panels": [ { "aliasColors": {}, @@ -267,11 +307,13 @@ "legend": { "alignAsTable": true, "avg": true, - "current": false, - "max": false, - "min": false, + "current": true, + "max": true, + "min": true, "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -319,6 +361,12 @@ "receivedTotalPacketsAccumulated" ], "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" } ] ], @@ -393,11 +441,13 @@ "legend": { "alignAsTable": true, "avg": true, - "current": false, - "max": false, - "min": false, + "current": true, + "max": true, + "min": true, "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -444,6 +494,12 @@ "receivedOctetsAccumulated" ], "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" } ] ], @@ -505,6 +561,246 @@ "showTitle": false, "title": "Dashboard Row", "titleSize": "h6" + }, + { + "collapse": false, + "height": 362, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + } + ], + "measurement": "memoryUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "memoryFree" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "disk" + ], + "type": "tag" + } + ], + "measurement": "diskUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "diskOpsWriteLast" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + }, + { + "condition": "AND", + "key": "disk", + "operator": "=", + "value": "sda" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" } ], "schemaVersion": 14, @@ -514,48 +810,16 @@ "list": [ { "allValue": null, - "current": { - "tags": [], - "text": "All", - "value": [ - "$__all" - ] - }, + "current": {}, "datasource": "VESEvents", "hide": 0, "includeAll": true, "label": "host", "multi": true, "name": "host", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "4C4C4544-004C-4E10-8054-B6C04F475831", - "value": "4C4C4544-004C-4E10-8054-B6C04F475831" - }, - { - "selected": false, - "text": "4C4C4544-004C-4E10-8057-B6C04F475831", - "value": "4C4C4544-004C-4E10-8057-B6C04F475831" - }, - { - "selected": false, - "text": "4C4C4544-004C-5010-8053-B6C04F475831", - "value": "4C4C4544-004C-5010-8053-B6C04F475831" - }, - { - "selected": false, - "text": "4C4C4544-004E-3110-8051-C3C04F445631", - "value": "4C4C4544-004E-3110-8051-C3C04F445631" - } - ], + "options": [], "query": "SHOW TAG VALUES WITH KEY=system", - "refresh": 0, + "refresh": 1, "regex": "", "sort": 0, "tagValuesQuery": "", @@ -592,6 +856,6 @@ }, "timezone": "browser", "title": "VES Demo", - "version": 2 + "version": 3 } } diff --git a/tools/monitor.py b/tools/monitor.py index 91c0eae..9405b48 100644 --- a/tools/monitor.py +++ b/tools/monitor.py @@ -291,6 +291,60 @@ def save_event(body): pdata = pdata[:i] + ' ' + pdata[i+1:] send_to_influxdb("systemLoad", pdata) +# "cpuUsageArray": [ +# { +# "cpuIdentifier": "15", +# "cpuIdle": 99.8998998999, +# "cpuUsageInterrupt": 0, +# "cpuUsageNice": 0, +# "cpuUsageSoftIrq": 0, +# "cpuUsageSteal": 0, +# "cpuUsageSystem": 0, +# "cpuUsageUser": 0.1001001001, +# "cpuWait": 0, +# "percentUsage": 0.0 +# }, + + if 'cpuUsageArray' in jobj['event']['measurementsForVfScalingFields']: + print('Found diskUsageArray') + for disk in jobj['event']['measurementsForVfScalingFields']['cpuUsageArray']: + id=disk['cpuIdentifier'] + pdata = 'cpuUsage,system={},cpu={}'.format(source,id) + d = disk.items() + for key,val in d: + if key != 'cpuIdentifier': + pdata = pdata + ',{}={}'.format(key,val) + i=pdata.find(',', pdata.find('cpu=')) + pdata = pdata[:i] + ' ' + pdata[i+1:] + send_to_influxdb("cpuUsage", pdata) + +# "diskUsageArray": [ +# { +# "diskIdentifier": "sda", +# "diskIoTimeLast": 0.3996139893, +# "diskMergedReadLast": 0, +# "diskMergedWriteLast": 26.1747155344, +# "diskOctetsReadLast": 0, +# "diskOctetsWriteLast": 309767.93302, +# "diskOpsReadLast": 0, +# "diskOpsWriteLast": 10.9893839563, +# "diskTimeReadLast": 0, +# "diskTimeWriteLast": 0.699324445683 +# }, + + if 'diskUsageArray' in jobj['event']['measurementsForVfScalingFields']: + print('Found diskUsageArray') + for disk in jobj['event']['measurementsForVfScalingFields']['diskUsageArray']: + id=disk['diskIdentifier'] + pdata = 'diskUsage,system={},disk={}'.format(source,id) + d = disk.items() + for key,val in d: + if key != 'diskIdentifier': + pdata = pdata + ',{}={}'.format(key,val) + i=pdata.find(',', pdata.find('disk=')) + pdata = pdata[:i] + ' ' + pdata[i+1:] + send_to_influxdb("diskUsage", pdata) + # "memoryUsageArray": [ # { # "memoryBuffered": 269056.0, diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh index 6e20bfd..7b6bddb 100644 --- a/tools/ves-setup.sh +++ b/tools/ves-setup.sh @@ -242,6 +242,7 @@ LoadPlugin csv StoreRates false +# TODO: complete the virt plugin install before enabling #LoadPlugin virt # # Connection "qemu:///system" @@ -275,7 +276,8 @@ LoadPlugin interface LoadPlugin memory LoadPlugin load LoadPlugin disk -LoadPlugin uuid +# TODO: how to set this option only to apply to VMs (not nodes) +#LoadPlugin uuid LoadPlugin write_kafka @@ -337,6 +339,8 @@ LoadPlugin match_regex EOF fi +# sudo sed -i -- "s/#Hostname \"localhost\"/Hostname \"$HOSTNAME\"/" /opt/collectd/etc/collectd.conf + if [[ $(grep -c $ves_hostname /etc/hosts) -eq 0 ]]; then log "add to /etc/hosts: $ves_kafka_host $ves_hostname" echo "$ves_kafka_host $ves_hostname" | sudo tee -a /etc/hosts -- cgit 1.2.3-korg