diff options
Diffstat (limited to 'monitor')
| -rw-r--r-- | monitor/automate_cadvisor_client.py | 37 | ||||
| -rw-r--r-- | monitor/automate_collectd_client.py | 38 | ||||
| -rw-r--r-- | monitor/automated-dashboard-datasource.py | 69 | ||||
| -rw-r--r-- | monitor/cadvisor_install.sh | 10 | ||||
| -rw-r--r-- | monitor/config/collectd-client.conf | 125 | ||||
| -rw-r--r-- | monitor/config/collectd.conf | 18 | ||||
| -rw-r--r-- | monitor/config/prometheus.yaml | 14 | ||||
| -rw-r--r-- | monitor/install-collectd-client.sh | 8 | ||||
| -rw-r--r-- | monitor/monitoring.sh | 34 |
9 files changed, 335 insertions, 18 deletions
diff --git a/monitor/automate_cadvisor_client.py b/monitor/automate_cadvisor_client.py new file mode 100644 index 00000000..95b98e9d --- /dev/null +++ b/monitor/automate_cadvisor_client.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2017 Huawei Tech and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging +import sys +import yaml +sys.path.insert(0, '/home/opnfv/bottlenecks/utils/infra_setup/passwordless_SSH/') +import ssh + +logger = logging.getLogger(__name__) +with open('/tmp/pod.yaml') as f: + dataMap = yaml.safe_load(f) + for x in dataMap: + for y in dataMap[x]: + if (y['role']=='Controller') or (y['role']=='Compute'): + ip = str(y['ip']) + user = str(y['user']) + pwd = str(y['password']) + ssh_d = ssh.SSH(user, host= ip, password= pwd) + status, stdout, stderr = ssh_d.execute("cd /etc && mkdir cadvisor-config") + if status: + raise Exception("Command failed with non-zero status.") + logger.info(stdout.splitlines()) + with open("/home/opnfv/bottlenecks/monitor/cadvisor_install.sh") as stdin_file: + ssh_d.run("cat > /etc/cadvisor-config/install.sh", stdin=stdin_file) + status, stdout, stderr = ssh_d.execute("sudo apt-get install docker.io") + if status: + raise Exception("Command for installing docker failed.") + logger.info(stdout.splitlines()) + ssh_d.run("cd /etc/cadvisor-config/ && bash ./install.sh") + diff --git a/monitor/automate_collectd_client.py b/monitor/automate_collectd_client.py new file mode 100644 index 00000000..6dd70676 --- /dev/null +++ b/monitor/automate_collectd_client.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2017 Huawei Tech and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging +import sys +import yaml +sys.path.insert(0, '/home/opnfv/bottlenecks/utils/infra_setup/passwordless_SSH/') +import ssh + +logger = logging.getLogger(__name__) +with open('/tmp/pod.yaml') as f: + dataMap = yaml.safe_load(f) + for x in dataMap: + for y in dataMap[x]: + if (y['role']=='Controller') or (y['role']=='Compute'): + ip = str(y['ip']) + user = str(y['user']) + pwd = str(y['password']) + ssh_d = ssh.SSH(user, host= ip, password= pwd) + status, stdout, stderr = ssh_d.execute("cd /etc && mkdir collectd-config") + if status: + raise Exception("Command failed with non-zero status.") + logger.info(stdout.splitlines()) + with open("/home/opnfv/bottlenecks/monitor/install-collectd-client.sh") as stdin_file: + ssh_d.run("cat > /etc/collectd-config/install.sh", stdin=stdin_file) + with open("/home/opnfv/bottlenecks/monitor/config/collectd-client.conf") as stdin_file: + ssh_d.run("cat > /etc/collectd-config/collectd.conf", stdin=stdin_file) + status, stdout, stderr = ssh_d.execute("sudo apt-get install docker.io") + if status: + raise Exception("Command for installing docker failed.") + logger.info(stdout.splitlines()) + ssh_d.run("cd /etc/collectd-config/ && bash ./install.sh") diff --git a/monitor/automated-dashboard-datasource.py b/monitor/automated-dashboard-datasource.py new file mode 100644 index 00000000..1e1b42f7 --- /dev/null +++ b/monitor/automated-dashboard-datasource.py @@ -0,0 +1,69 @@ +############################################################################## +# Copyright (c) 2017 Huawei Tech and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging +import socket +import requests +from oslo_serialization import jsonutils + + +logger = logging.getLogger(__name__) + + +def _create_dashboard(ip, port, path): + url = 'http://admin:admin@{}:{}/api/dashboards/db'.format(ip, port) + logger.info("Fetched IP for dashboard creation!") + with open(path) as f: + data = jsonutils.load(f) + try: + post(url, {"dashboard": data}) + logger.info( "Trying to post dashboard json!") + except Exception: + logger.info("Create dashboard failed") + raise + + +def _create_data_source(ip, port): + url = 'http://admin:admin@{}:{}/api/datasources'.format(ip, port) + logger.info("Fetched URL for datasource") + data = { + "name": "automated-ds", + "type": "prometheus", + "access": "direct", + "url": "http://{}:9090".format(ip), + } + try: + post(url, data) + logger.info("Trying to post datasource") + + except Exception: + logger.info("Create Datasources failed") + raise + + +def post(url, data): + data = jsonutils.dump_as_bytes(data) + logger.info("In post method for dumping data") + headers = {'Content-Type': 'application/json'} + try: + response = requests.post(url, data=data, headers=headers) + result = response.json() + logger.debug('The result is: %s', result) + logger.info("Trying to post") + return result + except Exception as e: + logger.info("Failed post" + str(e)) + raise + + +ip_address = socket.gethostbyname(socket.gethostname()) +_create_dashboard(ip_address, 3000, '/var/lib/grafana/' + + 'dashboards/' + + 'prometheus-system_rev1.json') +_create_data_source(ip_address, 3000) diff --git a/monitor/cadvisor_install.sh b/monitor/cadvisor_install.sh new file mode 100644 index 00000000..524e24d8 --- /dev/null +++ b/monitor/cadvisor_install.sh @@ -0,0 +1,10 @@ +sudo docker run \ + --volume=/:/rootfs:ro \ + --volume=/var/run:/var/run:rw \ + --volume=/sys:/sys:ro \ + --volume=/var/lib/docker/:/var/lib/docker:ro \ + --volume=/dev/disk/:/dev/disk:ro \ + --publish=8080:8080 \ + --detach=true \ + --name=cadvisor \ + google/cadvisor:v0.25.0 \ -storage_driver=Prometheus diff --git a/monitor/config/collectd-client.conf b/monitor/config/collectd-client.conf new file mode 100644 index 00000000..96a2a690 --- /dev/null +++ b/monitor/config/collectd-client.conf @@ -0,0 +1,125 @@ +# +# Config file for collectd(1). +# Please read collectd.conf(5) for a list of options. +# http://collectd.org/ +# + +############################################################################## +# Global # +#----------------------------------------------------------------------------# +# Global settings for the daemon. # +############################################################################## + +#Hostname "localhost" +#FQDNLookup true +#BaseDir "${prefix}/var/lib/collectd" +#PIDFile "${prefix}/var/run/collectd.pid" +#PluginDir "${exec_prefix}/lib/collectd" +#TypesDB "/usr/share/collectd/types.db" + +#----------------------------------------------------------------------------# +# When enabled, plugins are loaded automatically with the default options # +# when an appropriate <Plugin ...> block is encountered. # +# Disabled by default. # +#----------------------------------------------------------------------------# +#AutoLoadPlugin false + +#----------------------------------------------------------------------------# +# When enabled, internal statistics are collected, using "collectd" as the # +# plugin name. # +# Disabled by default. # +#----------------------------------------------------------------------------# +#CollectInternalStats false + +#----------------------------------------------------------------------------# +# Interval at which to query values. This may be overwritten on a per-plugin # +# base by using the 'Interval' option of the LoadPlugin block: # +# <LoadPlugin foo> # +# Interval 60 # +# </LoadPlugin> # +#----------------------------------------------------------------------------# +#Interval 10 + +#MaxReadInterval 86400 +#Timeout 2 +#ReadThreads 5 +#WriteThreads 5 + +# Limit the size of the write queue. Default is no limit. Setting up a limit is +# recommended for servers handling a high volume of traffic. +#WriteQueueLimitHigh 1000000 +#WriteQueueLimitLow 800000 + +############################################################################## +# Logging # +#----------------------------------------------------------------------------# +# Plugins which provide logging functions should be loaded first, so log # +# messages generated when loading or configuring other plugins can be # +# accessed. # +############################################################################## + +LoadPlugin syslog +#<Plugin syslog> +# LogLevel info +#</Plugin> + +############################################################################## +# LoadPlugin section # +#----------------------------------------------------------------------------# +# Lines beginning with a single `#' belong to plugins which have been built # +# but are disabled by default. # +# # +# Lines begnning with `##' belong to plugins which have not been built due # +# to missing dependencies or because they have been deactivated explicitly. # +############################################################################## + +LoadPlugin cpu +LoadPlugin interface +LoadPlugin memory +LoadPlugin network +LoadPlugin rrdtool +LoadPlugin write_http + +############################################################################## +# Plugin configuration # +#----------------------------------------------------------------------------# +# In this section configuration stubs for each plugin are provided. A desc- # +# ription of those options is available in the collectd.conf(5) manual page. # +############################################################################## + +#<Plugin cpu> +# ReportByCpu true +# ReportByState true +# ValuesPercentage false +#</Plugin> + +#<Plugin interface> +# Interface "eth0" +# IgnoreSelected false +#</Plugin> + +#<Plugin memory> +# ValuesAbsolute true +# ValuesPercentage false +#</Plugin> + +<Plugin network> + Server "192.168.121.2" "25826" +</Plugin> + +#<Plugin rrdtool> +# DataDir "${prefix}/var/lib/collectd/rrd" +# CreateFilesAsync false +# CacheTimeout 120 +# CacheFlush 900 +# WritesPerSecond 50 +#</Plugin> + +#<Plugin write_http> +# <Node "collectd_exporter"> +# URL "http://192.168.121.2:9103/collectd-post" +# Format "JSON" +# StoreRates false +# </Node> +#</Plugin> + diff --git a/monitor/config/collectd.conf b/monitor/config/collectd.conf index 62be9fbb..6be610e5 100644 --- a/monitor/config/collectd.conf +++ b/monitor/config/collectd.conf @@ -751,7 +751,7 @@ LoadPlugin write_http #</Plugin> <Plugin network> - Server "192.168.104.2" "25826" + Listen "192.168.121.2" "25826" </Plugin> #<Plugin network> @@ -1021,13 +1021,13 @@ LoadPlugin write_http # CollectStatistics true #</Plugin> -#<Plugin rrdtool> -# DataDir "${prefix}/var/lib/collectd/rrd" -# CreateFilesAsync false -# CacheTimeout 120 -# CacheFlush 900 -# WritesPerSecond 50 -#</Plugin> +<Plugin rrdtool> + DataDir "/var/lib/collectd/rrd" + CreateFilesAsync false + CacheTimeout 120 + CacheFlush 900 + WritesPerSecond 50 +</Plugin> #<Plugin sensors> # SensorConfigFile "/etc/sensors.conf" @@ -1325,7 +1325,7 @@ LoadPlugin write_http <Plugin write_http> <Node "collectd_exporter"> - URL "http://192.168.104.2:9103/collectd-post" + URL "http://192.168.121.2:9103/collectd-post" Format "JSON" StoreRates false </Node> diff --git a/monitor/config/prometheus.yaml b/monitor/config/prometheus.yaml index 35bf0401..3736d8e4 100644 --- a/monitor/config/prometheus.yaml +++ b/monitor/config/prometheus.yaml @@ -25,7 +25,15 @@ scrape_configs: scrape_interval: 5s static_configs: - - targets: ['192.168.104.2:9090'] + - targets: ['192.168.121.2:9090'] + + - job_name: 'cadvisor' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: + - targets: ['192.168.121.2:8080','10.1.0.50:8080','10.1.0.51:8080'] - job_name: 'collectd' @@ -33,7 +41,7 @@ scrape_configs: scrape_interval: 5s static_configs: - - targets: ['192.168.104.2:9103'] + - targets: ['192.168.121.2:9103'] - job_name: 'node' @@ -41,4 +49,4 @@ scrape_configs: scrape_interval: 5s static_configs: - - targets: ['192.168.104.2:9100']
\ No newline at end of file + - targets: ['192.168.121.2:9100'] diff --git a/monitor/install-collectd-client.sh b/monitor/install-collectd-client.sh new file mode 100644 index 00000000..00fa4c88 --- /dev/null +++ b/monitor/install-collectd-client.sh @@ -0,0 +1,8 @@ +MONITOR_CONFIG="/etc/collectd-config" + +# Collectd +sudo docker run --name bottlenecks-automated-collectd -d \ + --privileged \ + -v ${MONITOR_CONFIG}:/etc/collectd:ro \ + -v /proc:/mnt/proc:ro \ + fr3nd/collectd:5.5.0-1 diff --git a/monitor/monitoring.sh b/monitor/monitoring.sh index 8ad388a3..16cb3086 100644 --- a/monitor/monitoring.sh +++ b/monitor/monitoring.sh @@ -18,7 +18,7 @@ sudo docker run --name bottlenecks-node-exporter \ -v "/sys:/host/sys:ro" \ -v "/:/rootfs:ro" \ --net="host" \ - quay.io/prometheus/node-exporter \ + quay.io/prometheus/node-exporter:v0.14.0 \ -collector.procfs /host/proc \ -collector.sysfs /host/sys \ -collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc)($|/)" @@ -28,21 +28,43 @@ sudo docker run --name bottlenecks-collectd -d \ --privileged \ -v ${MONITOR_CONFIG}:/etc/collectd:ro \ -v /proc:/mnt/proc:ro \ - fr3nd/collectd + fr3nd/collectd:5.5.0-1 # Collectd-Exporter sudo docker run --name bottlenecks-collectd-exporter \ -d -p 9103:9103 \ - -p 25826:25826/udp prom/collectd-exporter \ + -p 25826:25826/udp prom/collectd-exporter:0.3.1 \ -collectd.listen-address=":25826" # Prometheus sudo docker run --name bottlenecks-prometheus \ -d -p 9090:9090 \ -v ${MONITOR_CONFIG}/prometheus.yaml:/etc/prometheus/prometheus.yml \ - prom/prometheus + prom/prometheus:v1.7.1 -$ Grafana +# Grafana sudo docker run --name bottlenecks-grafana \ -d -p 3000:3000 \ - grafana/grafana + -v ${GRAFANA}/config/grafana.ini:/etc/grafana/grafana.ini \ + grafana/grafana:4.5.0 + +# Cadvisor +sudo docker run \ + --volume=/:/rootfs:ro \ + --volume=/var/run:/var/run:rw \ + --volume=/sys:/sys:ro \ + --volume=/var/lib/docker/:/var/lib/docker:ro \ + --volume=/dev/disk/:/dev/disk:ro \ + --publish=8080:8080 \ + --detach=true \ + --name=cadvisor \ + google/cadvisor:v0.25.0 \ -storage_driver=Prometheus + +# Automate Collectd Client +python automate_collectd_client.py + +# Automate Cadvisor Client +python automate_cadvisor_client.py + +# Automate Prometheus Datasource and Grafana Dashboard creation +python automated-dashboard-datasource.py |