![\"Monitoring](\"https://wiki.opnfv.org/download/attachments/13212037/soak_monitoring.png\")
\n\n\nYou're using the monitoring suite developed by OPNFV Bottlenecks team for OPNFV long duration test.\nThe monitoring suite consists of a monitoring dashboard - \nGrafana ,\na monitoring data source -\nPrometheus and\nseveral monitoring tools -\nBarometer,\nCadvisor and\nNode.\nThe monitoring suite is evolving along with OPNFV releases and enriched features will be included based on practical needs.\n
", "editable": true, "error": false, "height": "100", "id": 91, "links": [], "mode": "html", "span": 7, "style": {}, "title": "", "transparent": true, "type": "text" }, { "content": "### OPNFV Long Duration Test\nThis is a collaborative initiative launched by OPNFV test working group (Testperf) for long duration stability test (soak test) in NFV realm.\nSeveral testing projects in OPNFV has been involved, i.e.,\n[Bottlenecks - system limit testing](https://wiki.opnfv.org/display/bottlenecks),\n[Functest - functional testing](https://wiki.opnfv.org/display/functest),\n[NFVbench -NFVI benchmark testing](https://wiki.opnfv.org/display/bottlenecks),\n[Storperf - storage testing](https://wiki.opnfv.org/display/bottlenecks),\n[VSperf - data path testing](https://wiki.opnfv.org/display/bottlenecks) and\n[Yardstick - performance testing](https://wiki.opnfv.org/display/bottlenecks).\nFor more detail, please refer to the [OPNFV LDT WIKI](https://wiki.opnfv.org/display/testing/Long+Duration+Testing).", "editable": true, "error": false, "id": 92, "links": [], "mode": "markdown", "span": 5, "style": {}, "title": "", "transparent": true, "type": "text" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Dashboard Information", "titleSize": "h5" }, { "collapse": false, "height": 177, "panels": [ { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "automated-ds", "decimals": 0, "editable": true, "error": false, "format": "s", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "height": "", "id": 24, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "30%", "prefix": "", "prefixFontSize": "20%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "span": 2, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": false }, "tableColumn": "", "targets": [ { "expr": "time() - node_boot_time", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 40 } ], "thresholds": "", "title": "Uptime", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "automated-ds", "editable": true, "error": false, "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "id": 31, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "span": 1, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": false }, "tableColumn": "", "targets": [ { "expr": "count(rate(container_last_seen{name=~\".+\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "refId": "A", "step": 40 } ], "thresholds": "", "title": "Containers", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], "datasource": "automated-ds", "decimals": 1, "editable": true, "error": false, "format": "percentunit", "gauge": { "maxValue": 1, "minValue": 0, "show": true, "thresholdLabels": false, "thresholdMarkers": true }, "id": 26, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "span": 2, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": false }, "tableColumn": "", "targets": [ { "expr": "min((node_filesystem_size{fstype=~\"xfs|ext4\"} - node_filesystem_free{fstype=~\"xfs|ext4\"} )/ node_filesystem_size{fstype=~\"xfs|ext4\"})", "format": "time_series", "hide": false, "intervalFactor": 2, "refId": "A", "step": 40 } ], "thresholds": "0.75, 0.90", "title": "Disk space", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], "datasource": "automated-ds", "decimals": 0, "editable": true, "error": false, "format": "percent", "gauge": { "maxValue": 100, "minValue": 0, "show": true, "thresholdLabels": false, "thresholdMarkers": true }, "id": 25, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "span": 2, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": false }, "tableColumn": "", "targets": [ { "expr": "((node_memory_MemTotal - node_memory_MemAvailable) / node_memory_MemTotal) * 100", "format": "time_series", "intervalFactor": 2, "refId": "A", "step": 40 } ], "thresholds": "70, 90", "title": "Memory", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "aliasColors": { "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}": "#BA43A9" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 3, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(rate(container_cpu_system_seconds_total[1m]))", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "a", "refId": "B", "step": 120 }, { "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m]))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 2, "legendFormat": "nur container", "refId": "F", "step": 4 }, { "expr": "sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m]))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 2, "legendFormat": "nur docker host", "metric": "", "refId": "A", "step": 4 }, { "expr": "sum(rate(process_cpu_seconds_total[$interval])) * 100", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "", "metric": "", "refId": "C", "step": 10 }, { "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m])) + sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m])) + sum(rate(process_cpu_seconds_total[1m]))", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "", "refId": "D", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": false, "values": [] }, "yaxes": [ { "format": "percent", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "Available Memory": "#7EB26D", "Unavailable Memory": "#7EB26D" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 38, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 2, "stack": true, "steppedLine": false, "targets": [ { "expr": "container_memory_rss{name=~\".+\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "D", "step": 20 }, { "expr": "sum(container_memory_rss{name=~\".+\"})", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "A", "step": 20 }, { "expr": "container_memory_usage_bytes{name=~\".+\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "B", "step": 20 }, { "expr": "container_memory_rss{id=\"/\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "C", "step": 20 }, { "expr": "sum(container_memory_rss)", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "E", "step": 20 }, { "expr": "node_memory_Buffers", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "node_memory_Dirty", "refId": "N", "step": 30 }, { "expr": "node_memory_MemFree", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "Free Memory", "refId": "F", "step": 20 }, { "expr": "node_memory_MemAvailable", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "Available Memory", "refId": "H", "step": 20 }, { "expr": "node_memory_MemTotal - node_memory_MemAvailable", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "Unavailable Memory", "refId": "G", "step": 20 }, { "expr": "node_memory_Inactive", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "I", "step": 2 }, { "expr": "node_memory_KernelStack", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "J", "step": 30 }, { "expr": "node_memory_Active", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "K", "step": 30 }, { "expr": "node_memory_MemTotal - (node_memory_Active + node_memory_MemFree + node_memory_Inactive)", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "Unknown", "refId": "L", "step": 40 }, { "expr": "node_memory_MemFree + node_memory_Inactive ", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "M", "step": 30 }, { "expr": "container_memory_rss{name=~\".+\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "O", "step": 30 }, { "expr": "node_memory_Inactive + node_memory_MemFree + node_memory_MemAvailable", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "", "refId": "P", "step": 40 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 10000000000 } ], "timeFrom": null, "timeShift": null, "title": "Available Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": false, "values": [] }, "yaxes": [ { "format": "bytes", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "headings": true, "id": 94, "limit": 10, "links": [], "query": "", "recent": true, "search": false, "span": 12, "starred": true, "tags": [], "title": "Status Quick Access", "type": "dashlist" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Nodes Overview - CPU, Network, Storage, Container Count", "titleSize": "h5" }, { "collapse": true, "height": 265, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 1.25 ], "type": "gt" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Panel Title alert", "noDataState": "keep_state", "notifications": [ { "id": 1 } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "decimals": 0, "editable": true, "error": false, "fill": 1, "id": 28, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "node_load1 / count by(job, instance)(count by(job, instance, cpu)(node_cpu))", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 10 }, { "expr": "count by(exported_instance, job)(collectd_load_0{exported_instance=~\"host.\"}) / (count by(job, exported_instance)(count by(exported_instance, job, cpu)(collectd_cpu{exported_instance=~\"host.\"})))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", "refId": "B", "step": 10 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.25 } ], "timeFrom": null, "timeShift": null, "title": "CPU Loads on Nodes", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": false, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": "1", "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "SENT": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 19, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(container_network_transmit_bytes_total[$interval])) by (instance) - sum(rate(container_network_receive_bytes_total[$interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "(SENT - RECEIVED) {{instance}}", "refId": "A", "step": 10 }, { "expr": "- sum(rate(container_network_transmit_bytes_total[$interval])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "SENT - {{instance}}", "refId": "B", "step": 10 }, { "expr": "sum(rate(container_network_receive_bytes_total[$interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "RECEIVED - {{instance}}", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Network Loads on Nodes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "transparent": false, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": false, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "IN on /sda": "#7EB26D", "OUT on /sda": "#890F02" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "-sum(rate(node_disk_bytes_read[$interval])) by (device)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "OUT on /{{device}}", "metric": "node_disk_bytes_read", "refId": "A", "step": 10 }, { "expr": "sum(rate(node_disk_bytes_written[$interval])) by (device)", "format": "time_series", "intervalFactor": 2, "legendFormat": "IN on /{{device}}", "metric": "", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk I/O", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": false, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Loads on Nodes - CPU, Network, Disk Loads Records", "titleSize": "h5" }, { "collapse": true, "height": 281, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(container_network_receive_bytes_total{name=~\".+\"}[$interval])) by (name)", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A", "step": 4 }, { "expr": "- rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])", "hide": true, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Received Network Traffic per Container", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "transparent": false, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A", "step": 4 }, { "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Sent Network Traffic per Container", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "transparent": false, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": "", "logBase": 10, "max": 8, "min": 0, "show": false } ] }, { "alert": { "conditions": [ { "evaluator": { "params": [ 850000000000 ], "type": "gt" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Free/Used Disk Space alert", "noDataState": "keep_state", "notifications": [ { "id": 1 } ] }, "aliasColors": { "Belegete Festplatte": "#BF1B00", "Free Disk Space": "#7EB26D", "Used Disk Space": "#7EB26D", "{}": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Used Disk Space", "yaxis": 1 } ], "spaceLength": 10, "span": 3, "stack": true, "steppedLine": false, "targets": [ { "expr": "node_filesystem_size{fstype=\"aufs\"} - node_filesystem_free{fstype=\"aufs\"}", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 10 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 850000000000 } ], "timeFrom": null, "timeShift": null, "title": "Used Disk Space", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": false, "values": [] }, "yaxes": [ { "format": "bytes", "label": "", "logBase": 1, "max": 1000000000000, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 5, "grid": {}, "id": 1, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 9, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\".+\"}[$interval])) by (name) * 100", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "metric": "", "refId": "F", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage per Container", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": "", "logBase": 1, "max": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 3, "grid": {}, "id": 10, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(container_memory_rss{name=~\".+\"}) by (name)", "hide": false, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A", "step": 4 }, { "expr": "container_memory_usage_bytes{name=~\".+\"}", "hide": true, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "B", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Usage per Container", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "editable": true, "error": false, "fill": 3, "grid": {}, "id": 34, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(container_memory_swap{name=~\".+\"}) by (name)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A", "step": 4 }, { "expr": "container_memory_usage_bytes{name=~\".+\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "B", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Swap per Container", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "fill": 1, "id": 95, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "count by(device)(container_fs_io_current)", "format": "time_series", "hide": true, "intervalFactor": 2, "refId": "A", "step": 120 }, { "expr": "container_fs_read_seconds_total{name=~\".+\"}", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "Read - {{name}}", "refId": "B", "step": 4 }, { "expr": "-container_fs_write_seconds_total{name=~\".+\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Write - {{name}}", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Container IO Seconds Total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "automated-ds", "fill": 1, "id": 96, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "container_fs_io_current{name=~\".+\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A", "step": 4 }, { "expr": "container_fs_read_seconds_total{name=~\".+\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "Read - {{name}}", "refId": "B", "step": 240 }, { "expr": "container_fs_write_seconds_total{name=~\".+\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "Write - {{name}}", "refId": "C", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Container IO Current", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Containers on Nodes - CPU, Traffic, Memory Usage", "titleSize": "h5" }, { "collapse": true, "height": null, "panels": [ { "columns": [], "datasource": "automated-ds", "editable": true, "error": false, "filterNull": false, "fontSize": "100%", "height": "400", "hideTimeOverride": false, "id": 93, "links": [], "pageSize": 100, "repeat": null, "scroll": true, "showHeader": true, "sort": { "col": 0, "desc": false }, "span": 9, "styles": [ { "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "decimals": 2, "pattern": "/.*/", "thresholds": [], "type": "number", "unit": "short" } ], "targets": [ { "expr": "cadvisor_version_info", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - cAdvisor Version: {{cadvisorVersion}} - Docker Version: {{dockerVersion}} - OS Version: {{osVersion}} - Host Kernel Version: {{kernelVersion}}", "refId": "A", "step": 4 }, { "expr": "prometheus_build_info", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Prometheus Version: {{version}} - Go Version: {{goversion}} - Prometheus Branch: {{branch}}", "refId": "B", "step": 4 }, { "expr": "node_exporter_build_info", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Node-Exporter Version: {{version}} - Go Version: {{goversion}} - Node Exporter Branch: {{branch}}", "refId": "C", "step": 4 }, { "expr": "collectd_exporter_build_info", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{instance}} - Collectd Exporter Version: {{version}} - Go Version: {{goversion}} - Collectd Exporter Branch: {{branch}} ", "refId": "G", "step": 4 } ], "timeFrom": null, "timeShift": null, "title": "Versions", "transform": "timeseries_aggregations", "type": "table" }, { "columns": [ { "text": "Avg", "value": "avg" } ], "datasource": "automated-ds", "editable": true, "error": false, "fontSize": "100%", "height": "400", "id": 36, "links": [], "pageSize": null, "scroll": true, "showHeader": true, "sort": { "col": 0, "desc": true }, "span": 3, "styles": [ { "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "decimals": 2, "pattern": "/.*/", "thresholds": [ "10000000", " 25000000" ], "type": "number", "unit": "decbytes" } ], "targets": [ { "expr": "sum(container_spec_memory_limit_bytes{name=~\".+\"} - container_memory_usage_bytes{name=~\".+\"}) by (name) ", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{name}}", "metric": "", "refId": "A", "step": 240 }, { "expr": "sum(container_spec_memory_limit_bytes{name=~\".+\"}) by (name) ", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "B", "step": 10 }, { "expr": "container_memory_usage_bytes{name=~\".+\"}", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "C", "step": 240 } ], "title": "Limit memory", "transform": "timeseries_aggregations", "type": "table" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Versions - Host OS, Docker, APP, Build Information", "titleSize": "h5" }, { "collapse": true, "height": 250, "panels": [], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "OVS Bridges on Nodes - TODO", "titleSize": "h5" }, { "collapse": true, "height": 250, "panels": [], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "VMs on Nodes - TODO", "titleSize": "h5" }, { "collapse": true, "height": 250, "panels": [], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Traffic Generators Status - TODO", "titleSize": "h5" } ], "schemaVersion": 14, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": ".+", "current": { "text": "All", "value": "$__all" }, "datasource": "automated-ds", "hide": 0, "includeAll": true, "label": "Container Group", "multi": true, "name": "containergroup", "options": [], "query": "label_values(container_group)", "refresh": 1, "regex": "", "sort": 0, "tagValuesQuery": null, "tags": [], "tagsQuery": null, "type": "query", "useTags": false }, { "auto": true, "auto_count": 50, "auto_min": "50s", "current": { "text": "auto", "value": "$__auto_interval" }, "datasource": null, "hide": 0, "includeAll": false, "label": "Interval", "multi": false, "name": "interval", "options": [ { "selected": true, "text": "auto", "value": "$__auto_interval" }, { "selected": false, "text": "30s", "value": "30s" }, { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "2m", "value": "2m" }, { "selected": false, "text": "3m", "value": "3m" }, { "selected": false, "text": "5m", "value": "5m" }, { "selected": false, "text": "7m", "value": "7m" }, { "selected": false, "text": "10m", "value": "10m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "14d", "value": "14d" }, { "selected": false, "text": "30d", "value": "30d" } ], "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "refresh": 2, "type": "interval" }, { "allValue": null, "current": { "text": null, "value": null }, "datasource": "automated-ds", "hide": 0, "includeAll": false, "label": "Node", "multi": true, "name": "server", "options": [], "query": "label_values(node_boot_time, instance)", "refresh": 1, "regex": "/([^:]+):.*/", "sort": 0, "tagValuesQuery": null, "tags": [], "tagsQuery": null, "type": "query", "useTags": false } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Stats Overview", "version": 2 }