Merge "docker: Added docker container for running collectd"

8 files changed, 887 insertions, 0 deletions

diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 00000000..0b4944e9
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,18 @@
+FROM centos:7
+
+RUN yum update -y
+RUN yum install -y which sudo
+ENV DOCKER y
+COPY systems /systems
+COPY src /src
+COPY 3rd_party /3rd_party
+COPY mibs /mibs
+
+WORKDIR /systems
+RUN sh ./build_base_machine.sh
+RUN useradd -ms /bin/bash collectd_exec
+RUN echo "collectd_exec ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+WORKDIR /src
+
+COPY docker/run_collectd.sh /run_collectd.sh
+RUN chmod +x /run_collectd.sh
diff --git a/docker/README b/docker/README
new file mode 100644
index 00000000..7f5633ac
--- /dev/null
+++ b/docker/README
@@ -0,0 +1,106 @@
+==============================================================================
+Readme for collectd docker container in barometer project
+==============================================================================
+
+This text file includes information about environment preparation and
+deployment collectd in docker container
+
+Table of content:
+1. DESCRIPTION
+2. SYSTEM REQUIREMENTS
+3. INSTALLATION NOTES
+4. ADDITIONAL STEPS
+
+------------------------------------------------------------------------------
+1. DESCRIPTION
+
+This Dockerfile provides instruction for building collect in isolated container
+
+------------------------------------------------------------------------------
+2. SYSTEM REQUIREMENTS
+
+  Docker >= 17.06.0-ce
+
+------------------------------------------------------------------------------
+3. INSTALLATION NOTES
+
+To build docker container run
+sudo docker build -f ./docker/Dockerfile .
+from barometer folder.
+
+To run builded image run
+sudo docker images
+Get docker image id
+sudo docker run -ti --net=host -v `pwd`/../src/collectd_sample_configs:/opt/collectd/etc/collectd.d \
+-v /var/run:/var/run -v /tmp:/tmp --privileged <image id> /run_collectd.sh
+
+To make some changes run
+sudo docker run -ti --net=host -v `pwd`/../collectd_sample_configs:/opt/collectd/etc/collectd.d \
+ -v /var/run:/var/run --privileged <image id> /bin/bash
+
+/opt/collectd/sbin/collectd -f
+
+------------------------------------------------------------------------------
+4. ADDITIONAL STEPS
+
+To check if container works properly additional packages should be installed
+on host system.
+
+MCELOG
+To simulate mcelog message use instruction in http://artifacts.opnfv.org/barometer/docs/index.html#mcelog-plugin
+
+git clone https://github.com/andikleen/mce-inject
+cd mce-inject/
+make
+sudo make install
+modprobe mce-inject
+
+go to mcelog folder
+sudo make test
+
+if runs multiple times mcelog service shoud be restarted(cause mcelog make test exits closes mcelog)
+
+VIRT
+http://artifacts.opnfv.org/barometer/docs/index.html#virt-plugin
+Check that libvirtd is running on the remote host
+systemctl status libvirtd
+virsh list
+virsh perf instance-00000003
+sudo virsh perf instance-00000003 --enable cpu_cycles --live
+sudo virsh perf instance-00000003 --enable cmt --live
+sudo virsh perf instance-00000003 --enable mbmt --live
+sudo virsh perf instance-00000003 --enable mbml --live
+sudo virsh perf instance-00000003 --enable instructions --live
+sudo virsh perf instance-00000003 --enable cache_references --live
+sudo virsh perf instance-00000003 --enable cache_mises --live
+sudo virsh perf instance-00000003 --enable cache_misses --live
+
+OVS
+To successfuly run ovs plugins in Docker you need an ovs instance to connect to
+
+sudo yum install -y openvswitch-switch
+sudo service openvswitch-switch start
+sudo ovs-vsctl set-manager ptcp:6640
+
+Alternatively you can build ovs from source
+yum -y install make gcc openssl-devel autoconf automake rpm-build \
+       redhat-rpm-config python-devel openssl-devel kernel-devel  \
+       kernel-debug-devel libtool wget python-six selinux-policy-devel
+mkdir -p ~/rpmbuild/SOURCES
+cd ~/rpmbuild/SOURCES
+wget http://openvswitch.org/releases/openvswitch-2.5.3.tar.gz
+tar xfz openvswitch-2.5.3.tar.gz
+sed 's/openvswitch-kmod, //g' rhel/openvswitch.spec > rhel/openvswitch_no_kmod.spec
+rpmbuild -bb --nocheck rhel/openvswitch_no_kmod.spec
+cd ../RPMS/x86_64/
+yum install -y openvswitch-2.5.3-1.x86_64.rpm
+sudo systemctl start openvswitch.service
+sudo ovs-vsctl set-manager ptcp:6640
+
+To check if connection is successfull please check
+sudo ovs-vsctl show
+319efc53-b321-49a9-b628-e8d70f9bd8a9
+    Manager "ptcp:6640"
+        is_connected: true - can be a marker that ovs plugins successfully connected
+    ovs_version: "2.5.3"
+on the host.
diff --git a/docker/configure_grafana.sh b/docker/configure_grafana.sh
new file mode 100755
index 00000000..059cca97
--- /dev/null
+++ b/docker/configure_grafana.sh
@@ -0,0 +1,18 @@
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+curl -u admin:admin -X POST -H 'content-type: application/json'\
+      http://127.0.0.1:3000/api/datasources -d \
+      '{"name":"collectd","type":"influxdb","url":"http://localhost:8086","access":"proxy","isDefault":true,"database":"collectd","user":"admin","password":"admin","basicAuth":false}'
+curl -u admin:admin -X POST -H 'content-type: application/json'\
+      http://127.0.0.1:3000/api/dashboards/db -d @cpu_usage_dashboard.json
diff --git a/docker/cpu_usage_dashboard.json b/docker/cpu_usage_dashboard.json
new file mode 100644
index 00000000..7eb0d7d0
--- /dev/null
+++ b/docker/cpu_usage_dashboard.json
@@ -0,0 +1,667 @@
+{
+  "dashboard": {
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": "-- Grafana --",
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "gnetId": null,
+    "graphTooltip": 0,
+    "hideControls": false,
+    "id": null,
+    "links": [
+
+    ],
+    "rows": [
+      {
+        "collapse": false,
+        "height": 250,
+        "panels": [
+          {
+            "aliasColors": {
+
+            },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "editable": true,
+            "error": false,
+            "fill": 1,
+            "grid": {
+
+            },
+            "id": 3,
+            "legend": {
+              "alignAsTable": true,
+              "avg": true,
+              "current": false,
+              "hideZero": true,
+              "max": true,
+              "min": true,
+              "rightSide": true,
+              "show": true,
+              "total": false,
+              "values": true
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [
+
+            ],
+            "nullPointMode": "connected",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [
+
+            ],
+            "spaceLength": 10,
+            "span": 12,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "alias": "idle",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'idle') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "B",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "idle"
+                  }
+                ]
+              },
+              {
+                "alias": "iowait",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'wait') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "C",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "wait"
+                  }
+                ]
+              },
+              {
+                "alias": "irq",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'interrupt') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "D",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "interrupt"
+                  }
+                ]
+              },
+              {
+                "alias": "nice",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'nice') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "E",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "nice"
+                  }
+                ]
+              },
+              {
+                "alias": "softirq",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'softirq') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "F",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "softirq"
+                  }
+                ]
+              },
+              {
+                "alias": "steal",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'steal') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "G",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "steal"
+                  }
+                ]
+              },
+              {
+                "alias": "system",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'system') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "H",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "system"
+                  }
+                ]
+              },
+              {
+                "alias": "user",
+                "dsType": "influxdb",
+                "expr": "",
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "intervalFactor": 2,
+                "measurement": "cpu_value",
+                "orderByTime": "ASC",
+                "policy": "default",
+                "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ \/^$host$\/ AND \"type_instance\" = 'user') AND $timeFilter GROUP BY time($interval) fill(null)",
+                "rawQuery": false,
+                "refId": "I",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [
+
+                      ],
+                      "type": "mean"
+                    }
+                  ]
+                ],
+                "tags": [
+                  {
+                    "key": "host",
+                    "operator": "=~",
+                    "value": "\/^$host$\/"
+                  },
+                  {
+                    "condition": "AND",
+                    "key": "type_instance",
+                    "operator": "=",
+                    "value": "user"
+                  }
+                ]
+              }
+            ],
+            "thresholds": [
+
+            ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "CPU Usage",
+            "tooltip": {
+              "msResolution": true,
+              "shared": true,
+              "sort": 0,
+              "value_type": "cumulative"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": [
+
+              ]
+            },
+            "yaxes": [
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              }
+            ]
+          }
+        ],
+        "repeat": null,
+        "repeatIteration": null,
+        "repeatRowId": null,
+        "showTitle": false,
+        "title": "Dashboard Row",
+        "titleSize": "h6"
+      }
+    ],
+    "schemaVersion": 14,
+    "style": "dark",
+    "tags": [
+
+    ],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "text": "default",
+            "value": "default"
+          },
+          "hide": 0,
+          "label": null,
+          "name": "datasource",
+          "options": [
+
+          ],
+          "query": "influxdb",
+          "refresh": 1,
+          "regex": "",
+          "type": "datasource"
+        },
+        {
+          "allValue": null,
+          "current": {
+            "text": "localhost",
+            "value": "localhost"
+          },
+          "datasource": "$datasource",
+          "hide": 0,
+          "includeAll": false,
+          "label": null,
+          "multi": false,
+          "name": "host",
+          "options": [
+
+          ],
+          "query": "SHOW TAG VALUES WITH KEY=host",
+          "refresh": 1,
+          "regex": "",
+          "sort": 0,
+          "tagValuesQuery": "",
+          "tags": [
+
+          ],
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        }
+      ]
+    },
+    "time": {
+      "from": "now-6h",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d"
+      ]
+    },
+    "timezone": "",
+    "title": "CPU Usage",
+    "version": 1
+  },
+  "overwrite": false
+}
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
new file mode 100644
index 00000000..b5029a45
--- /dev/null
+++ b/docker/docker-compose.yml
@@ -0,0 +1,16 @@
+version: '2'
+
+services:
+  influxdb:
+    image: influxdb:1.3.7
+    network_mode: host
+    volumes:
+      - /var/lib/influxdb
+      - ./influxdb.conf:/etc/influxdb/influxdb.conf
+      - ./types.db:/usr/share/collectd/types.db:ro
+  grafana:
+    image: grafana/grafana:4.6.0
+    network_mode: host
+    volumes:
+      - /var/lib/grafana
+
diff --git a/docker/get_types_db.sh b/docker/get_types_db.sh
new file mode 100755
index 00000000..7b9105e3
--- /dev/null
+++ b/docker/get_types_db.sh
@@ -0,0 +1,14 @@
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+wget https://raw.githubusercontent.com/collectd/collectd/collectd-5.8/src/types.db -O types.db
diff --git a/docker/influxdb.conf b/docker/influxdb.conf
new file mode 100644
index 00000000..69d69b18
--- /dev/null
+++ b/docker/influxdb.conf
@@ -0,0 +1,33 @@
+
+[meta]
+  dir = "/var/lib/influxdb/meta"
+
+[data]
+  dir = "/var/lib/influxdb/data"
+  wal-dir = "/var/lib/influxdb/wal"
+
+[coordinator]
+
+[retention]
+
+[shard-precreation]
+
+
+[monitor]
+
+
+[http]
+
+[[graphite]]
+
+[[collectd]]
+enabled = true
+typesdb = "/usr/share/collectd/types.db"
+[[opentsdb]]
+
+[[udp]]
+
+[continuous_queries]
+
+
+
diff --git a/docker/run_collectd.sh b/docker/run_collectd.sh
new file mode 100644
index 00000000..001cce1a
--- /dev/null
+++ b/docker/run_collectd.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+/opt/collectd/sbin/collectd -f