Merge "Support: Container Metrics collection and Visualization"

4 files changed, 1540 insertions, 0 deletions

diff --git a/conf/05_collector.conf b/conf/05_collector.conf
index b9974037..882ef414 100644
--- a/conf/05_collector.conf
+++ b/conf/05_collector.conf
@@ -66,3 +66,17 @@ MC_PROX_CMD = './runrapid.py'
 MC_PROX_OUT = 'RUNirq.irq.log'
 MC_CRON_OUT = '/tmp/ovs-cores.log'
 MC_BEAT_CFILE = '/etc/filebeat/filebeat.yml'
+
+###############################################
+# Cadvisor Specific configuration
+###############################################
+
+LOG_FILE_CADVISOR = 'cadvisor'
+CADVISOR_STORAGE_DRIVER = 'stdout,influxdb'
+# ip:port of influxdb
+CADVISOR_STORAGE_HOST = '10.10.120.22:8086'
+CADVISOR_DRIVER_DB = '_internal'
+# names of all containers to calcualte results
+#CADVISOR_CONTAINERS = ['container1name','container2name']
+CADVISOR_CONTAINERS = []
+
diff --git a/tools/collectors/cadvisor/__init__.py b/tools/collectors/cadvisor/__init__.py
new file mode 100755
index 00000000..235ab875
--- /dev/null
+++ b/tools/collectors/cadvisor/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2020 University Of Delhi.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Wrapper for cAdvisor as a collector
+"""
diff --git a/tools/collectors/cadvisor/cadvisor.py b/tools/collectors/cadvisor/cadvisor.py
new file mode 100644
index 00000000..de48cecd
--- /dev/null
+++ b/tools/collectors/cadvisor/cadvisor.py
@@ -0,0 +1,218 @@
+# Copyright 2020 University Of Delhi.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Collects container metrics from cAdvisor.
+Sends metrics to influxDB and also stores results locally.
+"""
+
+import subprocess
+import logging
+import os
+from collections import OrderedDict
+
+from tools.collectors.collector import collector
+from tools import tasks
+from conf import settings
+
+
+
+# inherit from collector.Icollector.
+class Cadvisor(collector.ICollector):
+    """A collector of container metrics based on cAdvisor
+
+    It starts cadvisor and collects metrics.
+    """
+
+    def __init__(self, results_dir, test_name):
+        """
+        Initialize collection of statistics
+        """
+        self._logger = logging.getLogger(__name__)
+        self.resultsdir = results_dir
+        self.testname = test_name
+        self._pid = 0
+        self._results = OrderedDict()
+        self._log = os.path.join(results_dir,
+                                 settings.getValue('LOG_FILE_CADVISOR') +
+                                 '_' + test_name + '.log')
+        self._logfile = 0
+
+
+    def start(self):
+        """
+        Starts collection of statistics by cAdvisor and stores them
+        into-
+        1. The file in directory with test results
+        2. InfluxDB result container
+        """
+
+        # CMD options for cAdvisor
+        cmd = ['sudo', '/opt/cadvisor/cadvisor',
+               '-storage_driver='+settings.getValue('CADVISOR_STORAGE_DRIVER'),
+               '-storage_driver_host='+settings.getValue('CADVISOR_STORAGE_HOST'),
+               '-storage_driver_db='+settings.getValue('CADVISOR_DRIVER_DB'),
+               '-housekeeping_interval=0.5s',
+               '-storage_driver_buffer_duration=1s'
+              ]
+
+        self._logfile = open(self._log, 'a')
+
+        self._pid = subprocess.Popen(map(os.path.expanduser, cmd), stdout=self._logfile, bufsize=0)
+        self._logger.info('Starting cAdvisor')
+
+
+
+    def stop(self):
+        """
+        Stops collection of metrics by cAdvisor and stores statistic
+        summary for each monitored container into self._results dictionary
+        """
+        try:
+            subprocess.check_output(["pidof", "cadvisor"])
+            tasks.run_task(['sudo', 'pkill', '--signal', '2', 'cadvisor'],
+                           self._logger, 'Stopping cAdvisor', True)
+        except subprocess.CalledProcessError:
+            self._logger.error('Failed to stop cAdvisor, maybe process does not exist')
+
+
+        self._logfile.close()
+        self._logger.info('cAdvisor log available at %s', self._log)
+
+        containers = settings.getValue('CADVISOR_CONTAINERS')
+        self._results = cadvisor_log_result(self._log, containers)
+
+
+    def get_results(self):
+        """Returns collected statistics.
+        """
+        return self._results
+
+    def print_results(self):
+        """Logs collected statistics.
+        """
+        for cnt in self._results:
+            logging.info("Container: %s", cnt)
+            for (key, value) in self._results[cnt].items():
+
+                postfix = ''
+
+                if key == 'cpu_cumulative_usage':
+                    key = 'CPU_usage'
+                    value = round(float(value) / 1000000000, 4)
+                    postfix = '%'
+
+                if key in ['memory_usage', 'memory_working_set']:
+                    value = round(float(value) / 1024 / 1024, 4)
+                    postfix = 'MB'
+
+                if key in ['rx_bytes', 'tx_bytes']:
+                    value = round(float(value) / 1024 / 1024, 4)
+                    postfix = 'mBps'
+
+                logging.info("         Statistic: %s Value: %s %s",
+                             str(key), str(value), postfix)
+
+
+def cadvisor_log_result(filename, containers):
+    """
+    Processes cAdvisor logfile and returns average results
+
+    :param filename: Name of cadvisor logfile
+    :param containers: List of container names
+
+    :returns: Result as average stats of Containers
+    """
+    result = OrderedDict()
+    previous = OrderedDict()
+    logfile = open(filename, 'r')
+    with logfile:
+        # for every line
+        for _, line in enumerate(logfile):
+            # skip lines having root '/' metrics
+            if line[0:7] == 'cName=/':
+                continue
+
+            # parse line into OrderedDict
+            tmp_res = parse_line(line)
+
+            cnt = tmp_res['cName']
+
+            # skip if cnt is not in container list
+            if cnt not in containers:
+                continue
+
+            # add metrics to result
+            if cnt not in result:
+                result[cnt] = tmp_res
+                previous[cnt] = tmp_res
+                result[cnt]['count'] = 1
+            else:
+                for field in tmp_res:
+
+                    if field in ['rx_errors', 'tx_errors', 'memory_usage', 'memory_working_set']:
+                        val = float(tmp_res[field])
+                    elif field in ['cpu_cumulative_usage', 'rx_bytes', 'tx_bytes']:
+                        val = float(tmp_res[field]) - float(previous[cnt][field])
+                    else:
+                        # discard remaining fields
+                        try:
+                            result[cnt].pop(field)
+                        except KeyError:
+                            continue
+                        continue
+
+                    result[cnt][field] = float(result[cnt][field]) + val
+
+                result[cnt]['count'] += 1
+                previous[cnt] = tmp_res
+
+    # calculate average results for containers
+    result = calculate_average(result)
+    return result
+
+
+def calculate_average(results):
+    """
+    Calculates average for container stats
+    """
+    for cnt in results:
+        for field in results[cnt]:
+            if field != 'count':
+                val = float(results[cnt][field])/results[cnt]['count']
+                results[cnt][field] = '{0:.2f}'.format(val)
+
+        results[cnt].pop('count')
+        #sort results
+        results[cnt] = OrderedDict(sorted(results[cnt].items()))
+
+    return results
+
+
+def parse_line(line):
+    """
+    Reads single line from cAdvisor logfile
+
+    :param line: single line as str
+
+    :returns: OrderedDict of line read
+    """
+    tmp_res = OrderedDict()
+    # split line into array of "key=value" metrics
+    metrics = line.split()
+    for metric in metrics:
+        key, value = metric.split('=')
+        tmp_res[key] = value
+
+    return tmp_res
diff --git a/tools/docker/results/grafana/dashboards/container_metrics_dashboard.json b/tools/docker/results/grafana/dashboards/container_metrics_dashboard.json
new file mode 100644
index 00000000..ef0b32a1
--- /dev/null
+++ b/tools/docker/results/grafana/dashboards/container_metrics_dashboard.json
@@ -0,0 +1,1291 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "hideControls": false,
+  "id": 3,
+  "links": [],
+  "refresh": "5s",
+  "rows": [
+    {
+      "collapse": false,
+      "height": 234,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "influxdb",
+          "description": "Total CPU usage of container",
+          "fill": 0,
+          "height": "",
+          "hideTimeOverride": false,
+          "id": 1,
+          "interval": "",
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "minSpan": null,
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "Total",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "hide": false,
+              "measurement": "cpu_usage_total",
+              "orderByTime": "ASC",
+              "policy": "monitor",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  },
+                  {
+                    "params": [
+                      " / 1000000000"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": "1m",
+          "timeShift": null,
+          "title": "Total Usage",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "short",
+              "label": "Cores",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ]
+        },
+        {
+          "aliasColors": {
+            "Core 0": "#0a50a1",
+            "Core 1": "#890f02",
+            "Core 2": "#f9934e",
+            "Core 3": "#3f6833"
+          },
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "influxdb",
+          "description": "CPU usage per core",
+          "fill": 0,
+          "height": "",
+          "hideTimeOverride": false,
+          "id": 2,
+          "interval": "",
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "minSpan": null,
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "scopedVars": {
+            "core": {
+              "selected": false,
+              "text": "0",
+              "value": "0"
+            }
+          },
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "Core $tag_instance",
+              "dsType": "influxdb",
+              "groupBy": [
+                {
+                  "params": [
+                    "instance"
+                  ],
+                  "type": "tag"
+                }
+              ],
+              "measurement": "cpu_usage_per_cpu",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "query": "SELECT difference(\"value\")  / 1000000000 FROM \"monitor\".\"cpu_usage_per_cpu\" WHERE (\"container_name\" =~ /^$container$/) AND $timeFilter GROUP BY \"instance\"",
+              "rawQuery": false,
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  },
+                  {
+                    "params": [
+                      " / 1000000000"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": "1m",
+          "timeShift": null,
+          "title": "Usage per Core",
+          "tooltip": {
+            "shared": false,
+            "sort": 1,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "short",
+              "label": "Cores",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ]
+        },
+        {
+          "aliasColors": {
+            "Kernel": "#890f02",
+            "User": "#0a50a1"
+          },
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "influxdb",
+          "description": "CPU usage per User/Kernel",
+          "fill": 0,
+          "height": "",
+          "hideTimeOverride": false,
+          "id": 3,
+          "interval": "",
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "minSpan": null,
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "User",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "hide": false,
+              "measurement": "cpu_usage_user",
+              "orderByTime": "ASC",
+              "policy": "monitor",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  },
+                  {
+                    "params": [
+                      " / 1000000000"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            },
+            {
+              "alias": "Kernel",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "cpu_usage_system",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "B",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  },
+                  {
+                    "params": [
+                      " / 1000000000"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": "1m",
+          "timeShift": null,
+          "title": "Usage Breakdown",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "short",
+              "label": "Cores",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "CPU",
+      "titleSize": "h3"
+    },
+    {
+      "collapse": false,
+      "height": 250,
+      "panels": [
+        {
+          "aliasColors": {
+            "Hot": "#890f02",
+            "Total": "#0a50a1"
+          },
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": null,
+          "description": "Memory Usage",
+          "fill": 1,
+          "height": "",
+          "id": 4,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": 250,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "Total",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "memory_usage",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [
+                      " / 1024/1024"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            },
+            {
+              "alias": "Hot",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "memory_working_set",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "B",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [
+                      " / 1024/1024"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": "1m",
+          "timeShift": null,
+          "title": "Memory",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "decmbytes",
+              "label": "Megabytes",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "Memory",
+      "titleSize": "h3"
+    },
+    {
+      "collapse": false,
+      "height": 250,
+      "panels": [
+        {
+          "aliasColors": {
+            "rx_bytes": "#890f02",
+            "tx_bytes": "#0a50a1"
+          },
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": null,
+          "description": "Rx/Tx",
+          "fill": 0,
+          "height": "",
+          "id": 5,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "rx_bytes",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "rx_bytes",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  },
+                  {
+                    "params": [
+                      " / 1024"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            },
+            {
+              "alias": "tx_bytes",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "tx_bytes",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "B",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  },
+                  {
+                    "params": [
+                      " / 1024"
+                    ],
+                    "type": "math"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": "1m",
+          "timeShift": null,
+          "title": "Throughput",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "KBs",
+              "label": "Kilobytes per second",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ]
+        },
+        {
+          "aliasColors": {
+            "rx_errors": "#890f02",
+            "tx_bytes": "#0a50a1",
+            "tx_errors": "#0a50a1"
+          },
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": null,
+          "fill": 0,
+          "height": "",
+          "id": 6,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "rx_errors",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "rx_errors",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            },
+            {
+              "alias": "tx_errors",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "tx_errors",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "B",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "difference"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": "1m",
+          "timeShift": null,
+          "title": "Errors",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": "Errors per second",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "Network",
+      "titleSize": "h3"
+    },
+    {
+      "collapse": false,
+      "height": 30,
+      "panels": [
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "#299c46",
+            "rgba(237, 129, 40, 0.89)",
+            "#d44a3a"
+          ],
+          "datasource": null,
+          "decimals": 2,
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "height": "200px",
+          "hideTimeOverride": false,
+          "id": 7,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 6,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": true
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "dsType": "influxdb",
+              "groupBy": [],
+              "measurement": "fs_usage",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "container_name",
+                  "operator": "=~",
+                  "value": "/^$container$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": "",
+          "timeFrom": "1m",
+          "title": "Storage usage",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "avg"
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "Storage",
+      "titleSize": "h3"
+    },
+    {
+      "collapse": false,
+      "height": 250,
+      "panels": [
+        {
+          "columns": [],
+          "datasource": "influxdb",
+          "description": "Runtime table",
+          "fontSize": "80%",
+          "height": "",
+          "id": 8,
+          "links": [],
+          "pageSize": 5,
+          "scroll": false,
+          "showHeader": true,
+          "sort": {
+            "col": 0,
+            "desc": true
+          },
+          "span": 12,
+          "styles": [
+            {
+              "alias": "Time",
+              "dateFormat": "MM/DD/YY h:mm:ss a",
+              "pattern": "Time",
+              "type": "date"
+            },
+            {
+              "alias": "",
+              "colorMode": null,
+              "colors": [
+                "rgba(245, 54, 54, 0.9)",
+                "rgba(237, 129, 40, 0.89)",
+                "rgba(50, 172, 45, 0.97)"
+              ],
+              "decimals": 2,
+              "pattern": "/.*/",
+              "thresholds": [],
+              "type": "number",
+              "unit": "short"
+            }
+          ],
+          "targets": [
+            {
+              "alias": "$col",
+              "dsType": "influxdb",
+              "groupBy": [],
+              "limit": "5",
+              "measurement": "runtime",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "Alloc"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "Frees"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "HeapAlloc"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "HeapIdle"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "HeapObjects"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "HeapReleased"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "HeapSys"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "Lookups"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "Mallocs"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "NumGC"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "NumGoroutine"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "PauseTotalNs"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "Sys"
+                    ],
+                    "type": "field"
+                  }
+                ],
+                [
+                  {
+                    "params": [
+                      "TotalAlloc"
+                    ],
+                    "type": "field"
+                  }
+                ]
+              ],
+              "tags": []
+            }
+          ],
+          "timeFrom": "1m",
+          "title": "Runtime Metrics",
+          "transform": "timeseries_to_columns",
+          "type": "table"
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": false,
+      "title": "Dashboard Row",
+      "titleSize": "h6"
+    }
+  ],
+  "schemaVersion": 14,
+  "style": "dark",
+  "tags": [
+    "vsperf",
+    "container"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "selected": false,
+          "tags": [],
+          "text": "nginx",
+          "value": "nginx"
+        },
+        "datasource": "influxdb",
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "container",
+        "options": [],
+        "query": "show tag values with key = container_name",
+        "refresh": 1,
+        "regex": "[a-zA-Z0-9_/]*",
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "tags": [],
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": "influxdb",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": false,
+        "name": "core",
+        "options": [],
+        "query": "show tag values with key = instance",
+        "refresh": 1,
+        "regex": "",
+        "sort": 3,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Container Metrics",
+  "version": 12
+}
\ No newline at end of file