diff options
Diffstat (limited to 'docker')
17 files changed, 338 insertions, 12 deletions
diff --git a/docker/ansible/collectd6_test.yml b/docker/ansible/collectd6_test.yml new file mode 100644 index 00000000..c1a3a8b4 --- /dev/null +++ b/docker/ansible/collectd6_test.yml @@ -0,0 +1,143 @@ +--- +# ansible-playbook -e PR=<PRID> -e new_plugin=<plugin> collectd6_test.yml + +# As well as passing a PRID, a config command should be passable too since +# a lot of the plugins have been explicitly disabled in the build. +- hosts: localhost + become: true + tasks: + - name: Set names for containers to be used for testing + set_fact: + collectd5_container_name: "bar-collectd-latest" + collectd6_container_name: "bar-collectd-6{{ '-' + PR if PR is defined }}" + flask_container_name: "test-collectd-5-v-6" + + - name: Remove existing containers + docker_container: + name: "{{ item }}" + state: absent + force_kill: yes + with_items: + - "{{ collectd5_container_name }}" + - "{{ collectd6_container_name }}" + - "{{ flask_container_name }}" + + - name: Get a list of containers + command: + docker ps -a + register: output + + - name: Confirm that existing test containers were removed + assert: + that: + - 'not "{{ collectd5_container_name }}" in output.stdout' + - 'not "{{ collectd6_container_name }}" in output.stdout' + - 'not "{{ flask_container_name }}" in output.stdout' + + - name: Build collectd containers + include_role: + name: build_collectd + args: + apply: + tags: + - latest + - collectd-6 + - flask_test + vars: + COLLECTD_PULL_REQUESTS: "{{ PR | default() }}" + COLLECTD_CONFIG_CMD_ARGS: "{{ '--enable-' + new_plugin if new_plugin is defined }}" + + - name: "Set up config for write_http plugin" + set_fact: + collectd_plugins: "{{ collectd_plugins | default([]) | union(['write_http']) }}" + collectd_plugin_write_http_nodes: + flask: + url: http://localhost:5000 + format: "Command" + + - name: Generate collectd configs + include_role: + name: config_files + + # Since I can't skip-tags here, I have to remove the plugins later + # TODO(efoley) Add a disable_plugins and enable_plugins list to + # roles/config_files, as an alternative to tags. + # This alternative is kinda needed anyway, so that it's easier to add extra + # plugins instead of using. + # ``{{ collectd_plugins | default([]) | union(['the_plugin_i_want_to_enable'])}}`` + # Tags can stay, since they are convenient, and easier to pass to the + # command line than a list of plugins + - name: "Remove plugin configs" + file: + path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf" + state: absent + with_items: + - snmp_agent + - intel_pmu + + # TODO(efoley): The path here should be parameterised, to a degree, I don't + # want it to be repeated. And I shouldn't assume that this is always going + # to be the value (unless it is in vars/main instead of defaults/main) + - name: "Remove plugin configs (collectd 6)" + file: + path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf" + state: absent + with_items: + - csv + - network + - rrdtool + - write_kafka + - write_prometheus + - logfile + + - debug: + var: PR + + - name: Run the collectd-6 container + include_role: + name: run_collectd + vars: + collectd_image_name: "opnfv/barometer-collectd-6{{ '-' + PR if PR is defined }}" + collectd_container_name: "{{ collectd6_container_name }}" + + - name: Run the collectd-latest container + include_role: + name: run_collectd + vars: + collectd_image_name: opnfv/barometer-collectd-latest + collectd_container_name: "{{ collectd5_container_name }}" + + - name: Run the flask test container + docker_container: + name: "{{ flask_container_name }}" + image: test-collectd-write_http + detach: yes + state: started + #network_mode: host + published_ports: 5000:5000 + + - name: Check output for flask app + become: true + shell: | + docker logs {{ flask_container_name }} {{ '| grep "' + new_plugin + '"' if new_plugin is defined }} | tail -200 + register: output + + - debug: + var: output.stdout_lines + + - name: Get a list of running containers + become: true + command: + docker ps + register: output + + - name: Make sure that the expected containers are running + assert: + that: + - '"{{ collectd6_container_name }}" in output.stdout' + - '"{{ collectd5_container_name }}" in output.stdout' + - '"{{ flask_container_name }}" in output.stdout' + +# Create a small report at the end for collectd versions... +# Update Apply PRs to check out a branch when it is a single PR +# OR update these playbooks to use the tag way of checking out a PR diff --git a/docker/ansible/collectd_build.yml b/docker/ansible/collectd_build.yml new file mode 100644 index 00000000..d5cad076 --- /dev/null +++ b/docker/ansible/collectd_build.yml @@ -0,0 +1,22 @@ +# Copyright 2021 Anuket and others +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ansible-playbook collectd_build.yml +# +- hosts: localhost + become: true + become_user: root + gather_facts: true + roles: + - name: build_collectd diff --git a/docker/ansible/roles/build_collectd/tasks/main.yml b/docker/ansible/roles/build_collectd/tasks/main.yml new file mode 100644 index 00000000..6faddde5 --- /dev/null +++ b/docker/ansible/roles/build_collectd/tasks/main.yml @@ -0,0 +1,71 @@ +# Copyright 2021 Anuket and others +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +- name: Build stable container + docker_image: + name: anuket/barometer-collectd + build: + path: "{{ playbook_dir }}/../barometer-collectd/" + source: build + tags: + - stable + +- name: Build the latest container + docker_image: + name: anuket/barometer-collectd-latest + build: + path: "{{ playbook_dir }}/../../" + dockerfile: "docker/barometer-collectd-latest/Dockerfile" + source: build + tags: + - latest + +- name: Build collectd-experimental + docker_image: + name: anuket/barometer-collectd-experimental + build: + path: "{{ playbook_dir }}/../../" + dockerfile: "docker/barometer-collectd-experimental/Dockerfile" + args: + COLLECTD_FLAVOR: experimental + COLLECTD_TAG: "{{ COLLECTD_TAG | default('main') }}" + COLLECTD_PULL_REQUESTS: "{{ COLLECTD_PULL_REQUESTS | default() }}" + source: build + tags: + - experimental + +- name: Build collectd-6 + docker_image: + name: "anuket/barometer-collectd-6{{ ( '-' + COLLECTD_PULL_REQUESTS ) if COLLECTD_PULL_REQUESTS is defined else '' }}" + build: + path: "{{ playbook_dir }}/../../" + dockerfile: "docker/barometer-collectd-experimental/Dockerfile" + args: + COLLECTD_FLAVOR: collectd-6 + COLLECTD_TAG: "{{ COLLECTD_TAG | default('collectd-6.0') }}" + COLLECTD_CONFIG_CMD_ARGS: "{{ COLLECTD_CONFIG_CMD_ARGS if COLLECTD_CONFIG_CMD_ARGS is defined }}" + source: build + tags: + - collectd-6 + +- name: Build test_app for write_http + docker_image: + name: test-collectd-write_http + build: + path: "{{ playbook_dir }}/../flask_app/" + source: build + tags: + - flask_test + - never + diff --git a/docker/ansible/roles/config_files/tasks/logparser.yml b/docker/ansible/roles/config_files/tasks/logparser.yml index 615d2e2b..72adcea2 100644 --- a/docker/ansible/roles/config_files/tasks/logparser.yml +++ b/docker/ansible/roles/config_files/tasks/logparser.yml @@ -1,4 +1,4 @@ -#Copyright 2019 OPNFV and Intel Corporation +# Copyright 2019-21 Anuket, Intel Corporation, and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,9 +15,8 @@ - name: enable logparser plugin template: - src: experimental/logparser.conf.j2 + src: logparser.conf.j2 dest: "{{ config_file_dir }}/logparser.conf" - when: flavor|default('stable')|string == 'experimental' tags: - logparser diff --git a/docker/ansible/roles/config_files/templates/experimental/logparser.conf.j2 b/docker/ansible/roles/config_files/templates/logparser.conf.j2 index 1f1a725b..1f1a725b 100644 --- a/docker/ansible/roles/config_files/templates/experimental/logparser.conf.j2 +++ b/docker/ansible/roles/config_files/templates/logparser.conf.j2 diff --git a/docker/ansible/roles/config_files/vars/main.yml b/docker/ansible/roles/config_files/vars/main.yml index 410e24ed..02fd7fb9 100644 --- a/docker/ansible/roles/config_files/vars/main.yml +++ b/docker/ansible/roles/config_files/vars/main.yml @@ -18,6 +18,9 @@ collectd_hostname: "{{ inventory_hostname }}" collectd_plugin_capabilities_port: "9564" +collectd_plugin_cpu_valuespercentage: False +collectd_plugin_cpu_reportbystate: True + collectd_plugin_csv_datadir: "{{ csv_log_dir }}" collectd_plugin_csv_storerates: False diff --git a/docker/ansible/roles/install_docker/tasks/fedora.yml b/docker/ansible/roles/install_docker/tasks/fedora.yml index 56eebd59..f2a4f403 100644 --- a/docker/ansible/roles/install_docker/tasks/fedora.yml +++ b/docker/ansible/roles/install_docker/tasks/fedora.yml @@ -24,7 +24,7 @@ packages: - python3-dnf - python3 - - libselinux-python + - python3-libselinux - name: set up docker repository command: "dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo && dnf makecache" diff --git a/docker/ansible/roles/run_collectd/tasks/main.yml b/docker/ansible/roles/run_collectd/tasks/main.yml index 2c5d0e67..bf5aabf5 100644 --- a/docker/ansible/roles/run_collectd/tasks/main.yml +++ b/docker/ansible/roles/run_collectd/tasks/main.yml @@ -15,7 +15,7 @@ - name: remove bar-collectd container docker_container: - name: bar-collectd + name: "{{ collectd_container_name }}" state: absent tags: - rm_containers @@ -52,7 +52,7 @@ - name: launch collectd container docker_container: - name: bar-collectd + name: "{{ collectd_container_name }}" image: "{{ collectd_image_name }}" volumes: "{{ volumes_list }}" entrypoint: "/run_collectd.sh" diff --git a/docker/ansible/roles/run_collectd/vars/main.yml b/docker/ansible/roles/run_collectd/vars/main.yml index 96c9032e..26007ecf 100644 --- a/docker/ansible/roles/run_collectd/vars/main.yml +++ b/docker/ansible/roles/run_collectd/vars/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- +collectd_container_name: "bar-collectd" default_flavor: "{{ flavor|default('stable')|string }}" flavor_image_name: "{{ 'barometer-collectd-latest' if (default_flavor == 'master' or default_flavor == 'latest') else diff --git a/docker/barometer-collectd-experimental/Dockerfile b/docker/barometer-collectd-experimental/Dockerfile index 0a6d9806..f051ef35 100644 --- a/docker/barometer-collectd-experimental/Dockerfile +++ b/docker/barometer-collectd-experimental/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM centos:8 as builder +FROM quay.io/centos/centos:stream8 as builder ARG COLLECTD_FLAVOR=experimental ARG COLLECTD_PULL_REQUESTS @@ -32,7 +32,7 @@ WORKDIR ${repos_dir}/barometer/systems RUN ./build_base_machine.sh && \ dnf clean all && rm -rf /var/cache/dnf -FROM centos:8 +FROM quay.io/centos/centos:stream8 COPY docker/barometer-collectd-experimental/run_collectd.sh /run_collectd.sh RUN chmod +x /run_collectd.sh diff --git a/docker/barometer-collectd-latest/Dockerfile b/docker/barometer-collectd-latest/Dockerfile index b3a7a646..4b6dfefb 100644 --- a/docker/barometer-collectd-latest/Dockerfile +++ b/docker/barometer-collectd-latest/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM centos:8 +FROM quay.io/centos/centos:stream8 ARG COLLECTD_FLAVOR=latest ARG WITH_DPDK=y diff --git a/docker/barometer-collectd/Dockerfile b/docker/barometer-collectd/Dockerfile index e263e97f..db155572 100644 --- a/docker/barometer-collectd/Dockerfile +++ b/docker/barometer-collectd/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM centos:8 +FROM quay.io/centos/centos:stream8 RUN dnf update -y && \ dnf install -y which sudo git-core && \ dnf clean all && \ diff --git a/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json b/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json index 0b8ab73e..bab2742d 100644 --- a/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json +++ b/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json @@ -582,7 +582,7 @@ "measurement": "cpu_value", "orderByTime": "ASC", "policy": "default", - "query": "select derivative(mean(value),1s) AS idle from \"cpu_value\" WHERE \"type\" = 'cpu' AND \"host\" =~ /^$host$/ AND $timeFilter GROUP BY time($interval), instance, type_instance fill(null)", + "query": "select derivative(mean(value),1s) AS idle from \"cpu_value\" WHERE \"host\" =~ /^$host$/ AND $timeFilter GROUP BY time($interval), instance, type_instance fill(null)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -710,7 +710,7 @@ "measurement": "cpu_value", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ /^$host$/ AND \"type\" = 'cpu' AND \"type_instance\" != 'idle') AND $timeFilter GROUP BY time($__interval), \"instance\", \"type_instance\" fill(none)", + "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ /^$host$/ AND \"type_instance\" != 'idle') AND $timeFilter GROUP BY time($__interval), \"instance\", \"type_instance\" fill(none)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", diff --git a/docker/flask_app/Dockerfile b/docker/flask_app/Dockerfile new file mode 100644 index 00000000..67e6d589 --- /dev/null +++ b/docker/flask_app/Dockerfile @@ -0,0 +1,25 @@ +# Copyright 2021 Anuket and others. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM python:3-alpine + +EXPOSE 5000 + +WORKDIR /app +COPY ./ /app + +RUN python -m pip install -r requirements.txt + +ENTRYPOINT ["python", "flask_app.py"] diff --git a/docker/flask_app/README b/docker/flask_app/README new file mode 100644 index 00000000..03f8f3ff --- /dev/null +++ b/docker/flask_app/README @@ -0,0 +1,45 @@ +To build this run: + sudo docker build -t my-flask-app . + +To run the app and see collectd metrics: + + sudo docker run -d --net=host my-flask-app + OR + sudo docker run -d -p 5000:5000 my-flask-app + +and configure collectd to use the write_http plugin: + + LoadPlugin write_http + + <Plugin "write_http"> + <Node "example"> + URL "http://127.0.0.1:5000" + Format Command + # Format JSON + </Node> + </Plugin> + +Format Command is used to make the output more readable for humans. +You can also use JSON. + +Later the server will do something more useful. +To view the metrics that are being sent by collectd, run:: + + sudo docker inspect <container_id> + #OR + sudo docker logs <container_id> + +Metrics from collectd-5.x will use PUTVAL +Metrics from collectd-6.x will use PUTMETRIC + +Sample output:: + + 127.0.0.1 - - [21/Apr/2021 19:31:49] "POST / HTTP/1.1" 200 - + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/gauge-TSC interval=10.000 1619029909.268:2112.02271161789 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/frequency-busy interval=10.000 1619029909.268:1613.51555288381 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/percent-c1 interval=10.000 1619029909.268:86.2353665532377 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/frequency-average interval=10.000 1619029909.268:222.094501460956 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-pkg00/temperature interval=10.000 1619029909.268:53 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-pkg00/temperature-tcc_activation interval=10.000 1619029909.268:100 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu04/frequency-average interval=10.000 1619029909.268:206.978572579757 + diff --git a/docker/flask_app/flask_app.py b/docker/flask_app/flask_app.py new file mode 100644 index 00000000..771a91bc --- /dev/null +++ b/docker/flask_app/flask_app.py @@ -0,0 +1,16 @@ +from flask import Flask, request +import json + +app = Flask(__name__) + +@app.route('/', methods=['GET', 'POST']) +def get_data(): + #print(request.data) + #print(type(request.data)) + print(request.data.decode('utf-8')) + #print(json.loads(request.data.decode("utf-8"))) + + return 'This is working!' + +if __name__=='__main__': + app.run(debug=True, host='0.0.0.0') diff --git a/docker/flask_app/requirements.txt b/docker/flask_app/requirements.txt new file mode 100644 index 00000000..e3e9a71d --- /dev/null +++ b/docker/flask_app/requirements.txt @@ -0,0 +1 @@ +Flask |