diff options
45 files changed, 783 insertions, 380 deletions
@@ -3,46 +3,36 @@ project: 'Barometer' project_creation_date: '' project_category: '' lifecycle_state: '' -project_lead: &opnfv_barometer_ptl +project_lead: &anuket_barometer_ptl name: 'Emma Foley' email: 'efoley@redhat.com' id: 'efoley' company: 'redhat.com' timezone: 'Europe/Dublin' -primary_contact: *opnfv_barometer_ptl +primary_contact: *anuket_barometer_ptl issue_tracking: type: 'jira' - url: 'https://jira.opnfv.org/projects/barometer' + url: 'https://jira.anuket.io/projects/barometer' key: 'barometer' mailing_list: - type: 'mailman2' - url: 'opnfv-tech-discuss@lists.opnfv.org' - tag: '[barometer]' + type: 'groups.io' + url: 'anuket-tech-discuss@lists.anuket.io' + tag: '#barometer' realtime_discussion: - type: irc - server: 'freenode.net' - channel: '#opnfv-barometer' + type: slack + server: 'anuketworkspace.slack.com' + channel: '#barometer' meetings: - - type: 'gotomeeting+irc' - agenda: # eg: 'https://wiki.opnfv.org/display/' - url: # eg: 'https://global.gotomeeting.com/join/819733085' - server: 'freenode.net' - channel: '#opnfv-meeting' + - type: 'zoom' + agenda: 'https://wiki.anuket.io/display/HOME/Meetings' + url: 'https://zoom.us/j/92950950555?pwd=K2tWZWovWXJWTmNuZHIxNGlZWkpHQT0' repeats: 'weekly' - time: # eg: '16:00 UTC' + time: '17:00 Europe/Dublin' repositories: - 'barometer' - 'puppet-barometer' committers: - - <<: *opnfv_barometer_ptl - - name: 'Maryam Tahhan' - email: 'maryam.tahhan@intel.com' - company: 'intel.com' - id: 'maryamtahhan' - - name: 'Calin Gherghe' - email: 'calin.gherghe@intel.com' - company: 'intel.com' - id: 'cgherghe' + - <<: *anuket_barometer_ptl - name: 'Al Morton' email: 'acmorton@att.com' company: 'att.com' @@ -51,10 +41,6 @@ committers: email: 'efoley@redhat.com' company: 'redhat.com' id: 'elfoley' - - name: 'Aaron Smith' - email: 'aasmith@redhat.com' - company: 'redhat.com' - id: 'TsaLaGi' - name: 'Matthias Runge' email: 'mrunge@redhat.com' company: 'redhat.com' diff --git a/docker/ansible/collectd6_test.yml b/docker/ansible/collectd6_test.yml new file mode 100644 index 00000000..c1a3a8b4 --- /dev/null +++ b/docker/ansible/collectd6_test.yml @@ -0,0 +1,143 @@ +--- +# ansible-playbook -e PR=<PRID> -e new_plugin=<plugin> collectd6_test.yml + +# As well as passing a PRID, a config command should be passable too since +# a lot of the plugins have been explicitly disabled in the build. +- hosts: localhost + become: true + tasks: + - name: Set names for containers to be used for testing + set_fact: + collectd5_container_name: "bar-collectd-latest" + collectd6_container_name: "bar-collectd-6{{ '-' + PR if PR is defined }}" + flask_container_name: "test-collectd-5-v-6" + + - name: Remove existing containers + docker_container: + name: "{{ item }}" + state: absent + force_kill: yes + with_items: + - "{{ collectd5_container_name }}" + - "{{ collectd6_container_name }}" + - "{{ flask_container_name }}" + + - name: Get a list of containers + command: + docker ps -a + register: output + + - name: Confirm that existing test containers were removed + assert: + that: + - 'not "{{ collectd5_container_name }}" in output.stdout' + - 'not "{{ collectd6_container_name }}" in output.stdout' + - 'not "{{ flask_container_name }}" in output.stdout' + + - name: Build collectd containers + include_role: + name: build_collectd + args: + apply: + tags: + - latest + - collectd-6 + - flask_test + vars: + COLLECTD_PULL_REQUESTS: "{{ PR | default() }}" + COLLECTD_CONFIG_CMD_ARGS: "{{ '--enable-' + new_plugin if new_plugin is defined }}" + + - name: "Set up config for write_http plugin" + set_fact: + collectd_plugins: "{{ collectd_plugins | default([]) | union(['write_http']) }}" + collectd_plugin_write_http_nodes: + flask: + url: http://localhost:5000 + format: "Command" + + - name: Generate collectd configs + include_role: + name: config_files + + # Since I can't skip-tags here, I have to remove the plugins later + # TODO(efoley) Add a disable_plugins and enable_plugins list to + # roles/config_files, as an alternative to tags. + # This alternative is kinda needed anyway, so that it's easier to add extra + # plugins instead of using. + # ``{{ collectd_plugins | default([]) | union(['the_plugin_i_want_to_enable'])}}`` + # Tags can stay, since they are convenient, and easier to pass to the + # command line than a list of plugins + - name: "Remove plugin configs" + file: + path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf" + state: absent + with_items: + - snmp_agent + - intel_pmu + + # TODO(efoley): The path here should be parameterised, to a degree, I don't + # want it to be repeated. And I shouldn't assume that this is always going + # to be the value (unless it is in vars/main instead of defaults/main) + - name: "Remove plugin configs (collectd 6)" + file: + path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf" + state: absent + with_items: + - csv + - network + - rrdtool + - write_kafka + - write_prometheus + - logfile + + - debug: + var: PR + + - name: Run the collectd-6 container + include_role: + name: run_collectd + vars: + collectd_image_name: "opnfv/barometer-collectd-6{{ '-' + PR if PR is defined }}" + collectd_container_name: "{{ collectd6_container_name }}" + + - name: Run the collectd-latest container + include_role: + name: run_collectd + vars: + collectd_image_name: opnfv/barometer-collectd-latest + collectd_container_name: "{{ collectd5_container_name }}" + + - name: Run the flask test container + docker_container: + name: "{{ flask_container_name }}" + image: test-collectd-write_http + detach: yes + state: started + #network_mode: host + published_ports: 5000:5000 + + - name: Check output for flask app + become: true + shell: | + docker logs {{ flask_container_name }} {{ '| grep "' + new_plugin + '"' if new_plugin is defined }} | tail -200 + register: output + + - debug: + var: output.stdout_lines + + - name: Get a list of running containers + become: true + command: + docker ps + register: output + + - name: Make sure that the expected containers are running + assert: + that: + - '"{{ collectd6_container_name }}" in output.stdout' + - '"{{ collectd5_container_name }}" in output.stdout' + - '"{{ flask_container_name }}" in output.stdout' + +# Create a small report at the end for collectd versions... +# Update Apply PRs to check out a branch when it is a single PR +# OR update these playbooks to use the tag way of checking out a PR diff --git a/src/collectd/collectd_sample_configs-latest/dpdkstat.conf b/docker/ansible/collectd_build.yml index 59906d4e..d5cad076 100644 --- a/src/collectd/collectd_sample_configs-latest/dpdkstat.conf +++ b/docker/ansible/collectd_build.yml @@ -1,4 +1,4 @@ -# Copyright 2017 OPNFV +# Copyright 2021 Anuket and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -LoadPlugin dpdkstat - -<Plugin dpdkstat> -# Coremask "0xf" -# ProcessType "secondary" -# FilePrefix "rte" - EnabledPortMask 0xffff -# PortName "interface1" -# PortName "interface2" -</Plugin> +--- +# ansible-playbook collectd_build.yml +# +- hosts: localhost + become: true + become_user: root + gather_facts: true + roles: + - name: build_collectd diff --git a/docker/ansible/roles/build_collectd/tasks/main.yml b/docker/ansible/roles/build_collectd/tasks/main.yml new file mode 100644 index 00000000..6faddde5 --- /dev/null +++ b/docker/ansible/roles/build_collectd/tasks/main.yml @@ -0,0 +1,71 @@ +# Copyright 2021 Anuket and others +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +- name: Build stable container + docker_image: + name: anuket/barometer-collectd + build: + path: "{{ playbook_dir }}/../barometer-collectd/" + source: build + tags: + - stable + +- name: Build the latest container + docker_image: + name: anuket/barometer-collectd-latest + build: + path: "{{ playbook_dir }}/../../" + dockerfile: "docker/barometer-collectd-latest/Dockerfile" + source: build + tags: + - latest + +- name: Build collectd-experimental + docker_image: + name: anuket/barometer-collectd-experimental + build: + path: "{{ playbook_dir }}/../../" + dockerfile: "docker/barometer-collectd-experimental/Dockerfile" + args: + COLLECTD_FLAVOR: experimental + COLLECTD_TAG: "{{ COLLECTD_TAG | default('main') }}" + COLLECTD_PULL_REQUESTS: "{{ COLLECTD_PULL_REQUESTS | default() }}" + source: build + tags: + - experimental + +- name: Build collectd-6 + docker_image: + name: "anuket/barometer-collectd-6{{ ( '-' + COLLECTD_PULL_REQUESTS ) if COLLECTD_PULL_REQUESTS is defined else '' }}" + build: + path: "{{ playbook_dir }}/../../" + dockerfile: "docker/barometer-collectd-experimental/Dockerfile" + args: + COLLECTD_FLAVOR: collectd-6 + COLLECTD_TAG: "{{ COLLECTD_TAG | default('collectd-6.0') }}" + COLLECTD_CONFIG_CMD_ARGS: "{{ COLLECTD_CONFIG_CMD_ARGS if COLLECTD_CONFIG_CMD_ARGS is defined }}" + source: build + tags: + - collectd-6 + +- name: Build test_app for write_http + docker_image: + name: test-collectd-write_http + build: + path: "{{ playbook_dir }}/../flask_app/" + source: build + tags: + - flask_test + - never + diff --git a/docker/ansible/roles/config_files/tasks/dpdk.yml b/docker/ansible/roles/config_files/tasks/dpdk.yml index a022de9e..2be146b0 100644 --- a/docker/ansible/roles/config_files/tasks/dpdk.yml +++ b/docker/ansible/roles/config_files/tasks/dpdk.yml @@ -12,24 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - -- name: enable dpdkstat plugin - template: - src: dpdkstat.conf.j2 - dest: "{{ config_file_dir }}/dpdkstat.conf" - tags: - # require this to be explicitly enabled - - never - - dpdkstat - -- name: enable dpdkevents plugin - template: - src: dpdkevents.conf.j2 - dest: "{{ config_file_dir }}/dpdkevents.conf" - tags: - - never - - dpdkevents - - name: enable dpdk_telemetry plugin set_fact: collectd_plugins: "{{ collectd_plugins | union(['dpdk_telemetry']) | unique }}" diff --git a/docker/ansible/roles/config_files/tasks/logparser.yml b/docker/ansible/roles/config_files/tasks/logparser.yml index 615d2e2b..72adcea2 100644 --- a/docker/ansible/roles/config_files/tasks/logparser.yml +++ b/docker/ansible/roles/config_files/tasks/logparser.yml @@ -1,4 +1,4 @@ -#Copyright 2019 OPNFV and Intel Corporation +# Copyright 2019-21 Anuket, Intel Corporation, and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,9 +15,8 @@ - name: enable logparser plugin template: - src: experimental/logparser.conf.j2 + src: logparser.conf.j2 dest: "{{ config_file_dir }}/logparser.conf" - when: flavor|default('stable')|string == 'experimental' tags: - logparser diff --git a/docker/ansible/roles/config_files/templates/dpdkevents.conf.j2 b/docker/ansible/roles/config_files/templates/dpdkevents.conf.j2 deleted file mode 100644 index 60bdb3e2..00000000 --- a/docker/ansible/roles/config_files/templates/dpdkevents.conf.j2 +++ /dev/null @@ -1,36 +0,0 @@ -#Copyright 2018 OPNFV and Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -<LoadPlugin dpdkevents> - Interval 1 -</LoadPlugin> - -<Plugin "dpdkevents"> -# <EAL> -# Coremask "0x1" -# MemoryChannels "4" -# FilePrefix "rte" -# </EAL> - <Event "link_status"> - SendEventsOnUpdate false - EnabledPortMask 0xffff - SendNotification true - </Event> -# <Event "keep_alive"> -# SendEventsOnUpdate false -# LCoreMask "0xf" -# KeepAliveShmName "/dpdk_keepalive_shm_name" -# SendNotification true -# </Event> -</Plugin> diff --git a/docker/ansible/roles/config_files/templates/dpdkstat.conf.j2 b/docker/ansible/roles/config_files/templates/dpdkstat.conf.j2 deleted file mode 100644 index 738fb4d0..00000000 --- a/docker/ansible/roles/config_files/templates/dpdkstat.conf.j2 +++ /dev/null @@ -1,25 +0,0 @@ -#Copyright 2018 OPNFV and Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -LoadPlugin dpdkstat - -<Plugin dpdkstat> -# Coremask "0xf" -# ProcessType "secondary" -# FilePrefix "rte" - EnabledPortMask 0xffff -# PortName "interface1" -# PortName "interface2" -</Plugin> - diff --git a/docker/ansible/roles/config_files/templates/experimental/logparser.conf.j2 b/docker/ansible/roles/config_files/templates/logparser.conf.j2 index 1f1a725b..1f1a725b 100644 --- a/docker/ansible/roles/config_files/templates/experimental/logparser.conf.j2 +++ b/docker/ansible/roles/config_files/templates/logparser.conf.j2 diff --git a/docker/ansible/roles/config_files/vars/main.yml b/docker/ansible/roles/config_files/vars/main.yml index 410e24ed..02fd7fb9 100644 --- a/docker/ansible/roles/config_files/vars/main.yml +++ b/docker/ansible/roles/config_files/vars/main.yml @@ -18,6 +18,9 @@ collectd_hostname: "{{ inventory_hostname }}" collectd_plugin_capabilities_port: "9564" +collectd_plugin_cpu_valuespercentage: False +collectd_plugin_cpu_reportbystate: True + collectd_plugin_csv_datadir: "{{ csv_log_dir }}" collectd_plugin_csv_storerates: False diff --git a/docker/ansible/roles/install_docker/tasks/fedora.yml b/docker/ansible/roles/install_docker/tasks/fedora.yml index 56eebd59..f2a4f403 100644 --- a/docker/ansible/roles/install_docker/tasks/fedora.yml +++ b/docker/ansible/roles/install_docker/tasks/fedora.yml @@ -24,7 +24,7 @@ packages: - python3-dnf - python3 - - libselinux-python + - python3-libselinux - name: set up docker repository command: "dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo && dnf makecache" diff --git a/docker/ansible/roles/run_collectd/tasks/main.yml b/docker/ansible/roles/run_collectd/tasks/main.yml index 2c5d0e67..bf5aabf5 100644 --- a/docker/ansible/roles/run_collectd/tasks/main.yml +++ b/docker/ansible/roles/run_collectd/tasks/main.yml @@ -15,7 +15,7 @@ - name: remove bar-collectd container docker_container: - name: bar-collectd + name: "{{ collectd_container_name }}" state: absent tags: - rm_containers @@ -52,7 +52,7 @@ - name: launch collectd container docker_container: - name: bar-collectd + name: "{{ collectd_container_name }}" image: "{{ collectd_image_name }}" volumes: "{{ volumes_list }}" entrypoint: "/run_collectd.sh" diff --git a/docker/ansible/roles/run_collectd/vars/main.yml b/docker/ansible/roles/run_collectd/vars/main.yml index ddfec146..26007ecf 100644 --- a/docker/ansible/roles/run_collectd/vars/main.yml +++ b/docker/ansible/roles/run_collectd/vars/main.yml @@ -1,4 +1,4 @@ -#Copyright 2019 OPNFV and Intel Corporation +# Copyright 2019-21 Anuket, Intel Corporation and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,11 @@ # limitations under the License. --- +collectd_container_name: "bar-collectd" default_flavor: "{{ flavor|default('stable')|string }}" flavor_image_name: "{{ 'barometer-collectd-latest' if (default_flavor == 'master' or default_flavor == 'latest') else 'barometer-collectd-experimental' if (default_flavor == 'experimental') else 'barometer-collectd-6' if (default_flavor == 'collectd-6') else 'barometer-collectd' }}" -collectd_image_name: "{{ 'opnfv/' + flavor_image_name }}" +collectd_image_name: "{{ 'anuket/' + flavor_image_name }}" diff --git a/docker/ansible/roles/run_grafana/tasks/main.yml b/docker/ansible/roles/run_grafana/tasks/main.yml index 0ec307fa..d4a1638c 100644 --- a/docker/ansible/roles/run_grafana/tasks/main.yml +++ b/docker/ansible/roles/run_grafana/tasks/main.yml @@ -1,4 +1,4 @@ -#Copyright 2018 OPNFV and Intel Corporation +# Copyright 2018-21 Anuket, Intel Corporation and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ - name: Remove barometer-grafana image docker_image: state: absent - name: opnfv/barometer-grafana + name: anuket/barometer-grafana tags: - rm_images - rm_grafana_image @@ -40,7 +40,7 @@ - name: launch barometer-grafana container docker_container: name: bar-grafana - image: opnfv/barometer-grafana + image: anuket/barometer-grafana volumes: - /var/lib/grafana:/var/lib/grafana ports: diff --git a/docker/ansible/roles/run_influxdb/tasks/main.yml b/docker/ansible/roles/run_influxdb/tasks/main.yml index 53187fae..02eeb788 100644 --- a/docker/ansible/roles/run_influxdb/tasks/main.yml +++ b/docker/ansible/roles/run_influxdb/tasks/main.yml @@ -1,4 +1,4 @@ -#Copyright 2018 OPNFV and Intel Corporation +# Copyright 2018-21 Anuket, Intel Corporation and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,9 +22,9 @@ - rm_influxdb_cont when: rm_containers|default(false)|bool == true -- name: Remove opnfv/barometer-influxdb image +- name: Remove barometer-influxdb image docker_image: - name: opnfv/barometer-influxdb + name: anuket/barometer-influxdb state: absent tags: - rm_images @@ -34,7 +34,7 @@ - name: launch barometer-influxdb container docker_container: name: bar-influxdb - image: opnfv/barometer-influxdb + image: anuket/barometer-influxdb volumes: - /var/lib/influxdb:/var/lib/influxdb exposed: diff --git a/docker/ansible/roles/run_kafka/tasks/main.yml b/docker/ansible/roles/run_kafka/tasks/main.yml index 34d1f910..f30acd89 100644 --- a/docker/ansible/roles/run_kafka/tasks/main.yml +++ b/docker/ansible/roles/run_kafka/tasks/main.yml @@ -1,4 +1,4 @@ -#Copyright 2018 OPNFV and Intel Corporation +# Copyright 2018-21 Anuket, Intel Corporation and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,19 +22,19 @@ - remove_bar-kafka when: rm_containers|default(false)|bool == true -- name: Remove opnfv/barometer-kafka image +- name: Remove anuket/barometer-kafka image docker_image: - name: opnfv/barometer-kafka + name: anuket/barometer-kafka state: absent tags: - remove_images - remove_kafka_image when: rm_images|default(false)|bool == true -- name: launch opnfv/barometer-kafka container +- name: launch anuket/barometer-kafka container docker_container: name: bar-kafka - image: opnfv/barometer-kafka + image: anuket/barometer-kafka env: zookeeper_node: "{{ zookeeper_hostname }}" broker_id: "{{ broker_id }}" diff --git a/docker/ansible/roles/run_ves/tasks/main.yml b/docker/ansible/roles/run_ves/tasks/main.yml index aa9c29d3..a203fa98 100644 --- a/docker/ansible/roles/run_ves/tasks/main.yml +++ b/docker/ansible/roles/run_ves/tasks/main.yml @@ -1,4 +1,4 @@ -#Copyright 2018 OPNFV and Intel Corporation +# Copyright 2018-21 Anuket, Intel Corporation and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,19 +22,19 @@ - remove_bar-ves when: rm_containers|default(false)|bool == true -- name: Remove opnfv/barometer-ves image +- name: Remove anuket/barometer-ves image docker_image: state: absent - name: opnfv/barometer-ves + name: anuket/barometer-ves tags: - remove_images - remove_ves_image when: rm_images|default(false)|bool == true -- name: launch opnfv/barometer-ves container +- name: launch anuket/barometer-ves container docker_container: name: bar-ves - image: opnfv/barometer-ves + image: anuket/barometer-ves detach: yes state: started restart: yes diff --git a/docker/barometer-collectd-experimental/Dockerfile b/docker/barometer-collectd-experimental/Dockerfile index 0a6d9806..f051ef35 100644 --- a/docker/barometer-collectd-experimental/Dockerfile +++ b/docker/barometer-collectd-experimental/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM centos:8 as builder +FROM quay.io/centos/centos:stream8 as builder ARG COLLECTD_FLAVOR=experimental ARG COLLECTD_PULL_REQUESTS @@ -32,7 +32,7 @@ WORKDIR ${repos_dir}/barometer/systems RUN ./build_base_machine.sh && \ dnf clean all && rm -rf /var/cache/dnf -FROM centos:8 +FROM quay.io/centos/centos:stream8 COPY docker/barometer-collectd-experimental/run_collectd.sh /run_collectd.sh RUN chmod +x /run_collectd.sh diff --git a/docker/barometer-collectd-latest/Dockerfile b/docker/barometer-collectd-latest/Dockerfile index b3a7a646..4b6dfefb 100644 --- a/docker/barometer-collectd-latest/Dockerfile +++ b/docker/barometer-collectd-latest/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM centos:8 +FROM quay.io/centos/centos:stream8 ARG COLLECTD_FLAVOR=latest ARG WITH_DPDK=y diff --git a/docker/barometer-collectd/Dockerfile b/docker/barometer-collectd/Dockerfile index e263e97f..db155572 100644 --- a/docker/barometer-collectd/Dockerfile +++ b/docker/barometer-collectd/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM centos:8 +FROM quay.io/centos/centos:stream8 RUN dnf update -y && \ dnf install -y which sudo git-core && \ dnf clean all && \ diff --git a/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json b/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json index 0b8ab73e..bab2742d 100644 --- a/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json +++ b/docker/barometer-grafana/dashboards/cpu_usage_dashboard.json @@ -582,7 +582,7 @@ "measurement": "cpu_value", "orderByTime": "ASC", "policy": "default", - "query": "select derivative(mean(value),1s) AS idle from \"cpu_value\" WHERE \"type\" = 'cpu' AND \"host\" =~ /^$host$/ AND $timeFilter GROUP BY time($interval), instance, type_instance fill(null)", + "query": "select derivative(mean(value),1s) AS idle from \"cpu_value\" WHERE \"host\" =~ /^$host$/ AND $timeFilter GROUP BY time($interval), instance, type_instance fill(null)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -710,7 +710,7 @@ "measurement": "cpu_value", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ /^$host$/ AND \"type\" = 'cpu' AND \"type_instance\" != 'idle') AND $timeFilter GROUP BY time($__interval), \"instance\", \"type_instance\" fill(none)", + "query": "SELECT mean(\"value\") FROM \"cpu_value\" WHERE (\"host\" =~ /^$host$/ AND \"type_instance\" != 'idle') AND $timeFilter GROUP BY time($__interval), \"instance\", \"type_instance\" fill(none)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", diff --git a/src/collectd/collectd_sample_configs/dpdkstat.conf b/docker/flask_app/Dockerfile index 919e6e6e..67e6d589 100644 --- a/src/collectd/collectd_sample_configs/dpdkstat.conf +++ b/docker/flask_app/Dockerfile @@ -1,24 +1,25 @@ -# Copyright 2017 OPNFV +# Copyright 2021 Anuket and others. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -LoadPlugin dpdkstat +# + +FROM python:3-alpine + +EXPOSE 5000 + +WORKDIR /app +COPY ./ /app -<Plugin dpdkstat> -# Coremask "0xf" -# ProcessType "secondary" -# FilePrefix "rte" - EnabledPortMask 0xffff -# PortName "interface1" -# PortName "interface2" -</Plugin> +RUN python -m pip install -r requirements.txt +ENTRYPOINT ["python", "flask_app.py"] diff --git a/docker/flask_app/README b/docker/flask_app/README new file mode 100644 index 00000000..03f8f3ff --- /dev/null +++ b/docker/flask_app/README @@ -0,0 +1,45 @@ +To build this run: + sudo docker build -t my-flask-app . + +To run the app and see collectd metrics: + + sudo docker run -d --net=host my-flask-app + OR + sudo docker run -d -p 5000:5000 my-flask-app + +and configure collectd to use the write_http plugin: + + LoadPlugin write_http + + <Plugin "write_http"> + <Node "example"> + URL "http://127.0.0.1:5000" + Format Command + # Format JSON + </Node> + </Plugin> + +Format Command is used to make the output more readable for humans. +You can also use JSON. + +Later the server will do something more useful. +To view the metrics that are being sent by collectd, run:: + + sudo docker inspect <container_id> + #OR + sudo docker logs <container_id> + +Metrics from collectd-5.x will use PUTVAL +Metrics from collectd-6.x will use PUTMETRIC + +Sample output:: + + 127.0.0.1 - - [21/Apr/2021 19:31:49] "POST / HTTP/1.1" 200 - + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/gauge-TSC interval=10.000 1619029909.268:2112.02271161789 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/frequency-busy interval=10.000 1619029909.268:1613.51555288381 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/percent-c1 interval=10.000 1619029909.268:86.2353665532377 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/frequency-average interval=10.000 1619029909.268:222.094501460956 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-pkg00/temperature interval=10.000 1619029909.268:53 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-pkg00/temperature-tcc_activation interval=10.000 1619029909.268:100 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu04/frequency-average interval=10.000 1619029909.268:206.978572579757 + diff --git a/docker/flask_app/flask_app.py b/docker/flask_app/flask_app.py new file mode 100644 index 00000000..771a91bc --- /dev/null +++ b/docker/flask_app/flask_app.py @@ -0,0 +1,16 @@ +from flask import Flask, request +import json + +app = Flask(__name__) + +@app.route('/', methods=['GET', 'POST']) +def get_data(): + #print(request.data) + #print(type(request.data)) + print(request.data.decode('utf-8')) + #print(json.loads(request.data.decode("utf-8"))) + + return 'This is working!' + +if __name__=='__main__': + app.run(debug=True, host='0.0.0.0') diff --git a/docker/flask_app/requirements.txt b/docker/flask_app/requirements.txt new file mode 100644 index 00000000..e3e9a71d --- /dev/null +++ b/docker/flask_app/requirements.txt @@ -0,0 +1 @@ +Flask diff --git a/docs/release/release-notes/index.rst b/docs/release/release-notes/index.rst index 6185fc97..3f676753 100644 --- a/docs/release/release-notes/index.rst +++ b/docs/release/release-notes/index.rst @@ -12,6 +12,7 @@ Anuket Barometer Release Notes :maxdepth: 1 unreleased + lakelse-release-notes kali-release-notes old-release-notes diff --git a/docs/release/release-notes/lakelse-release-notes.rst b/docs/release/release-notes/lakelse-release-notes.rst new file mode 100644 index 00000000..bf04342e --- /dev/null +++ b/docs/release/release-notes/lakelse-release-notes.rst @@ -0,0 +1,164 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) Anuket and others. + +============================ +Lakelse Release Notes +============================ + +.. _Release Notes_lakelse: + +Lakelse +======= + +.. _Release Notes_lakelse_Release Summary: + +Release Summary +--------------- + +.. docs/release/release-notes/notes/lakelse/add_unix_sock-e29efe16156c5c8e.yaml @ None + +Added unixsock plugin to one-click install. + + +.. docs/release/release-notes/notes/lakelse/ansible-build-containers-b4a4cc9cb70f83b3.yaml @ None + +Add ansible playbook for building the containers locally. + + +.. docs/release/release-notes/notes/lakelse/anuket_containers-21b4206cb26c9975.yaml @ None + +Since the anuket dockerhub repository was created, and containers are being pushed to there, instructions and build scripts have been updated to reflect this. + + +.. docs/release/release-notes/notes/lakelse/collectd-5-v-6-testing-cc821b32bad2794c.yaml @ None + +Testing playbooks were added to compare collectd5 vs collectd6, for the purpose of helping to review new PRs by comparing the generated metrics between versions. + + +.. docs/release/release-notes/notes/lakelse/remove_dpdk_stats_events_plugins-59f366855f6e4261.yaml @ None + +Remove dpdkstats and dpdkevents from Barometer. + + +.. docs/release/release-notes/notes/lakelse/update_logparser_config-0db3d2746e6ad582.yaml @ None + +Enable the Logparser plugin by default when using one-click install. + + +.. _Release Notes_lakelse_Testing Notes: + +Testing Notes +------------- + +.. docs/release/release-notes/notes/lakelse/collectd-5-v-6-testing-cc821b32bad2794c.yaml @ None + +- Added a playbook to compare collectd 5 and collectd 6. The playbook uses + existing ansible roles to build both collectd 5 and collectd 6 container + images, creates a common configuration, then runs the containers and shows + the outputs to let the user inspect the metrics and whether they match. + + +.. _Release Notes_lakelse_Documentation Updates: + +Documentation Updates +--------------------- + +.. docs/release/release-notes/notes/lakelse/anuket_containers-21b4206cb26c9975.yaml @ None + +- Docs have been updated to use anuket/ repository in dockerhub. + Container build instructions now use anuket/ prefix to tag images. + + +.. _Release Notes_lakelse_Container updates: + +Container updates +----------------- + +.. docs/release/release-notes/notes/lakelse/anuket_containers-21b4206cb26c9975.yaml @ None + +- Containers are now pulled from anuket/ repository in dockerhub. + +.. docs/release/release-notes/notes/lakelse/collectd-6-testing-flask-app-2bb0ca1326775dd8.yaml @ None + +- Add a flask app for testing collectd using metrics sent via write_http plugin. + +.. docs/release/release-notes/notes/lakelse/update-grafana-9bee82ecfa11f54a.yaml @ None + +- Grafana container was updated to support both jiffies and percent for cpu metrics. + + +.. _Release Notes_lakelse_Ansible playbook updates: + +Ansible playbook updates +------------------------ + +.. docs/release/release-notes/notes/lakelse/add_unix_sock-e29efe16156c5c8e.yaml @ None + +- Added `unixsock <https://collectd.org/documentation/manpages/collectd-unixsock.5.shtml>`_ + plugin to one-click install, which allows the user to interact with collectd using the + ``collectdctl`` command in the bar-collectd-* containers. + The unixsock plugin is useful for debugging issues in collectd, and can + be used to verify that metrics are being collected without having to + create CSV files or log into the container. + +.. docs/release/release-notes/notes/lakelse/ansible-build-containers-b4a4cc9cb70f83b3.yaml @ None + +- Added a playbook and role for building the collectd containers locally. + This automates the actions described in the docker install guide. The + ``barometer-collectd``, ``barometer-collectd-latest`` and the + ``barometer-collectd-experimental`` containers are now easier to build + locally. The ``barometer-collectd-6`` and + ``barometer-collectd-experimental`` containers can also be built with + arbirtary PRs applied, to aid in testing locally. + +.. docs/release/release-notes/notes/lakelse/anuket_containers-21b4206cb26c9975.yaml @ None + +- Containers are now pulled from anuker/ repository in dockerhub. + +.. docs/release/release-notes/notes/lakelse/update_logparser_config-0db3d2746e6ad582.yaml @ None + +- The logparser plugin is now rendered for all flavours. + The Logparser plugin has been part of collectd since 5.11, however, the ansible playbooks had it marked as experimental, and would not deploy it by default. + + +.. _Release Notes_lakelse_Build script updates: + +Build script updates +-------------------- + +.. docs/release/release-notes/notes/lakelse/update-apply-pr-script-46e6d547d331c5f2.yaml @ None + +- Update collectd_apply_pull_request.sh to rebase only if multiple chanegs are selected. The script will checkout the PR branch if there's only one PR_ID passed. + + +.. _Release Notes_lakelse_Normal Bug Fixes: + +Normal Bug Fixes +---------------- + +.. docs/release/release-notes/notes/lakelse/update-grafana-9bee82ecfa11f54a.yaml @ None + +- Update the grafana dashboard to show metrics in both jffies and percent, depending on what is configured. + + +.. _Release Notes_lakelse_Deprecations: + +Deprecations +------------ + +.. docs/release/release-notes/notes/lakelse/remove_dpdk_stats_events_plugins-59f366855f6e4261.yaml @ None + +- The dpdkstats and dpdkevents plugins were removed from Barometer. These + plugins are still available in collectd, however, will not be deployed by + Barometer. It is recommended that the DPDK telemetry plugin be used instead. + + +.. _Release Notes_lakelse_Other Notes: + +Other Notes +----------- + +.. docs/release/release-notes/notes/lakelse/add-reno-12eb20e3448b663b.yaml @ None + +- Add `reno <https://docs.openstack.org/reno/latest/index.html#>`_ and corresponding tox jobs (compile notes and add new notes) to make compiling release notes easier diff --git a/docs/release/release-notes/notes/add-reno-12eb20e3448b663b.yaml b/docs/release/release-notes/notes/add-reno-12eb20e3448b663b.yaml index c0e8c7e9..2456c099 100644 --- a/docs/release/release-notes/notes/add-reno-12eb20e3448b663b.yaml +++ b/docs/release/release-notes/notes/add-reno-12eb20e3448b663b.yaml @@ -4,4 +4,4 @@ documentation: Release notes are now automatically generated and included in the documentation using `reno <https://docs.openstack.org/reno/latest/index.html>`_. other: - | - Add `reno <https://docs.openstack.org/reno/latest/index.html#>`_ and corresponding tox jobs to make compiling release notes easier + Add `reno <https://docs.openstack.org/reno/latest/index.html#>`_ and corresponding tox jobs (compile notes and add new notes) to make compiling release notes easier diff --git a/docs/release/release-notes/notes/ansible-build-containers-b4a4cc9cb70f83b3.yaml b/docs/release/release-notes/notes/ansible-build-containers-b4a4cc9cb70f83b3.yaml new file mode 100644 index 00000000..aae4b999 --- /dev/null +++ b/docs/release/release-notes/notes/ansible-build-containers-b4a4cc9cb70f83b3.yaml @@ -0,0 +1,11 @@ +release_summary: > + Add ansible playbook for building the containers locally. +ansible: + - | + Added a playbook and role for building the collectd containers locally. + This automates the actions described in the docker install guide. The + ``barometer-collectd``, ``barometer-collectd-latest`` and the + ``barometer-collectd-experimental`` containers are now easier to build + locally. The ``barometer-collectd-6`` and + ``barometer-collectd-experimental`` containers can also be built with + arbirtary PRs applied, to aid in testing locally. diff --git a/docs/release/release-notes/notes/anuket_containers-21b4206cb26c9975.yaml b/docs/release/release-notes/notes/anuket_containers-21b4206cb26c9975.yaml new file mode 100644 index 00000000..75e7e4f0 --- /dev/null +++ b/docs/release/release-notes/notes/anuket_containers-21b4206cb26c9975.yaml @@ -0,0 +1,12 @@ +release_summary: > + Since the anuket dockerhub repository was created, and containers are being pushed to there, instructions and build scripts have been updated to reflect this. +docs: + - | + Docs have been updated to use anuket/ repository in dockerhub. + Container build instructions now use anuket/ prefix to tag images. +containers: + - | + Containers are now pulled from anuket/ repository in dockerhub. +ansible: + - | + Containers are now pulled from anuker/ repository in dockerhub. diff --git a/docs/release/release-notes/notes/collectd-5-v-6-testing-cc821b32bad2794c.yaml b/docs/release/release-notes/notes/collectd-5-v-6-testing-cc821b32bad2794c.yaml new file mode 100644 index 00000000..20013147 --- /dev/null +++ b/docs/release/release-notes/notes/collectd-5-v-6-testing-cc821b32bad2794c.yaml @@ -0,0 +1,10 @@ +release_summary: > + Testing playbooks were added to compare collectd5 vs collectd6, for the + purpose of helping to review new PRs by comparing the generated metrics + between versions. +testing: + - | + Added a playbook to compare collectd 5 and collectd 6. The playbook uses + existing ansible roles to build both collectd 5 and collectd 6 container + images, creates a common configuration, then runs the containers and shows + the outputs to let the user inspect the metrics and whether they match. diff --git a/docs/release/release-notes/notes/collectd-6-testing-flask-app-2bb0ca1326775dd8.yaml b/docs/release/release-notes/notes/collectd-6-testing-flask-app-2bb0ca1326775dd8.yaml new file mode 100644 index 00000000..9c605876 --- /dev/null +++ b/docs/release/release-notes/notes/collectd-6-testing-flask-app-2bb0ca1326775dd8.yaml @@ -0,0 +1,3 @@ +containers: + - | + Add a flask app for testing collectd using metrics sent via write_http plugin. diff --git a/docs/release/release-notes/notes/remove_dpdk_stats_events_plugins-59f366855f6e4261.yaml b/docs/release/release-notes/notes/remove_dpdk_stats_events_plugins-59f366855f6e4261.yaml new file mode 100644 index 00000000..78ab1c4c --- /dev/null +++ b/docs/release/release-notes/notes/remove_dpdk_stats_events_plugins-59f366855f6e4261.yaml @@ -0,0 +1,8 @@ +--- +release_summary: > + Remove dpdkstats and dpdkevents from Barometer. +deprecations: + - | + The dpdkstats and dpdkevents plugins were removed from Barometer. These + plugins are still available in collectd, however, will not be deployed by + Barometer. It is recommended that the DPDK telemetry plugin be used instead. diff --git a/docs/release/release-notes/notes/update-apply-pr-script-46e6d547d331c5f2.yaml b/docs/release/release-notes/notes/update-apply-pr-script-46e6d547d331c5f2.yaml new file mode 100644 index 00000000..de1be994 --- /dev/null +++ b/docs/release/release-notes/notes/update-apply-pr-script-46e6d547d331c5f2.yaml @@ -0,0 +1,3 @@ +build: + - | + Update collectd_apply_pull_request.sh to rebase only if multiple chanegs are selected. The script will checkout the PR branch if there's only one PR_ID passed. diff --git a/docs/release/release-notes/notes/update-grafana-9bee82ecfa11f54a.yaml b/docs/release/release-notes/notes/update-grafana-9bee82ecfa11f54a.yaml new file mode 100644 index 00000000..95e2cbdb --- /dev/null +++ b/docs/release/release-notes/notes/update-grafana-9bee82ecfa11f54a.yaml @@ -0,0 +1,6 @@ +containers: + - | + Grafana container was updated to support both jiffies and percent for cpu metrics. +fixes: + - | + Update the grafana dashboard to show metrics in both jffies and percent, depending on what is configured. diff --git a/docs/release/release-notes/notes/update_logparser_config-0db3d2746e6ad582.yaml b/docs/release/release-notes/notes/update_logparser_config-0db3d2746e6ad582.yaml new file mode 100644 index 00000000..e5be3eff --- /dev/null +++ b/docs/release/release-notes/notes/update_logparser_config-0db3d2746e6ad582.yaml @@ -0,0 +1,6 @@ +release_summary: > + Enable the Logparser plugin by default when using one-click install. +ansible: + - | + The logparser plugin is now rendered for all flavours. + The Logparser plugin has been part of collectd since 5.11, however, the ansible playbooks had it marked as experimental, and would not deploy it by default. diff --git a/docs/release/userguide/feature.userguide.rst b/docs/release/userguide/feature.userguide.rst index 02f8bdaa..2750bd8d 100644 --- a/docs/release/userguide/feature.userguide.rst +++ b/docs/release/userguide/feature.userguide.rst @@ -22,26 +22,15 @@ to support thresholding and notification. Barometer has enabled the following collectd plugins: -* *dpdkstat plugin*: A read plugin that retrieves stats from the DPDK extended - NIC stats API. - -* *dpdkevents plugin*: A read plugin that retrieves DPDK link status and DPDK - forwarding cores liveliness status (DPDK Keep Alive). - * *dpdk_telemetry plugin*: A read plugin to collect dpdk interface stats and - application or global stats from dpdk telemetry library. Both 'dpdkstat' and - 'dpdk_telemetry' plugins provides dpdk NIC Stats, but only 'dpdk_telemetry' - provides the DPDK Application stats. So in other words, 'dpdk_telemetry' is - an advanced version of dpdkstat. This plugin don't deal with dpdk events. - So not in related with 'dpdkevents' plugin. The mimimum dpdk version required - to use this plugin is 19.08. + application or global stats from dpdk telemetry library. The ``dpdk_telemetry`` + plugin provides both DPDK NIC Stats and DPDK application stats. + This plugin doesn't deal with dpdk events. + The mimimum dpdk version required to use this plugin is 19.08. .. note:: - dpdpkstat and dpdk_telemetry should not be used together. Use dpdk_telemetry - if your version of dpdk supports it (i.e. DPDK >= 19.08) and use dpdkstat otherwise. - dpdkstat, dpdkevents and dpdk_telemetry plugins should only be used if your dpdk - application doesn't already have more relevant metrics available(e.g.ovs_stats). - + The ``dpdk_telemetry`` plugin should only be used if your dpdk application + doesn't already have more relevant metrics available (e.g.ovs_stats). * `gnocchi plugin`_: A write plugin that pushes the retrieved stats to Gnocchi. It's capable of pushing any stats read through collectd to @@ -175,15 +164,15 @@ collectd, check out the `collectd-openstack-plugins GSG`_. Below is the per plugin installation and configuration guide, if you only want to install some/particular plugins. -DPDK plugins -^^^^^^^^^^^^^ +DPDK telemetry plugin +^^^^^^^^^^^^^^^^^^^^^ Repo: https://github.com/collectd/collectd Branch: main -Dependencies: `DPDK <https://www.dpdk.org/>`_ +Dependencies: `DPDK <https://www.dpdk.org/>`_ (runtime), libjansson (compile-time) -.. note:: DPDK statistics plugin requires DPDK version 16.04 or later. +.. note:: DPDK telemetry plugin requires DPDK version 19.08 or later. To build and install DPDK to /usr please see: https://github.com/collectd/collectd/blob/main/docs/BUILD.dpdkstat.md @@ -199,53 +188,35 @@ Building and installing collectd: $ make $ sudo make install -.. note:: If DPDK was installed in a non standard location you will need to - specify paths to the header files and libraries using *LIBDPDK_CPPFLAGS* and - *LIBDPDK_LDFLAGS*. You will also need to add the DPDK library symbols to the - shared library path using *ldconfig*. Note that this update to the shared - library path is not persistant (i.e. it will not survive a reboot). - -Example of specifying custom paths to DPDK headers and libraries: - -.. code:: bash - - $ ./configure LIBDPDK_CPPFLAGS="path to DPDK header files" LIBDPDK_LDFLAGS="path to DPDK libraries" - This will install collectd to default folder ``/opt/collectd``. The collectd configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. -To configure the dpdkstats plugin you need to modify the configuration file to -include (uncomment lines as appropriate): -.. literalinclude:: ../../../src/collectd/collectd_sample_configs/dpdkstat.conf - :start-at: LoadPlugin - :language: bash +To configure the dpdk_telemetry plugin you need to modify the configuration file to +include: -To configure the dpdkevents plugin you need to modify the configuration file to -include (uncomment lines as appropriate): +.. code:: bash -.. literalinclude:: ../../../src/collectd/collectd_sample_configs/dpdkevents.conf - :start-at: LoadPlugin - :language: bash + LoadPlugin dpdk_telemetry + <Plugin dpdk_telemetry> + #ClientSocketPath "/var/run/.client" + #DpdkSocketPath "/var/run/dpdk/rte/telemetry" + </Plugin> -.. note:: Currently, the DPDK library doesn’t support API to de-initialize - the DPDK resources allocated on the initialization. It means, the collectd - plugin will not be able to release the allocated DPDK resources - (locks/memory/pci bindings etc.) correctly on collectd shutdown or reinitialize - the DPDK library if primary DPDK process is restarted. The only way to release - those resources is to terminate the process itself. For this reason, the plugin - forks off a separate collectd process. This child process becomes a secondary - DPDK process which can be run on specific CPU cores configured by user through - collectd configuration file (“Coremask” EAL configuration option, the - hexadecimal bitmask of the cores to run on). +The plugin uses default values (as shown) for the socket paths, if you use different values, +uncomment and update ``ClientSocketPath`` and ``DpdkSocketPath`` as required. For more information on the plugin parameters, please see: https://github.com/collectd/collectd/blob/main/src/collectd.conf.pod -.. note:: dpdkstat plugin initialization time depends on read interval. It - requires 5 read cycles to set up internal buffers and states, during that time - no statistics are submitted. Also, if plugin is running and the number of DPDK - ports is increased, internal buffers are resized. That requires 3 read cycles - and no port statistics are submitted during that time. +.. note:: + + To gather metrics from a DPDK application, telemetry needs to be enabled. + This can be done by setting the ``CONFIG_RTE_LIBRTE_TELEMETRY=y`` config flag. + The application then needs to be run with the ``--telemetry`` EAL option, e.g. + :: + $dpdk/app/testpmd --telemetry -l 2,3,4 -n 4 + +For more information on the ``dpdk_telemetry`` plugin, see the `anuket wiki <https://wiki.anuket.io/display/HOME/DPDK+Telemetry+Plugin>`_. The Address-Space Layout Randomization (ASLR) security feature in Linux should be disabled, in order for the same hugepage memory mappings to be present in all @@ -270,27 +241,6 @@ To fully enable ASLR: For more information on multi-process support, please see: https://doc.dpdk.org/guides/prog_guide/multi_proc_support.html -**DPDK stats plugin limitations:** - -1. The DPDK primary process application should use the same version of DPDK - that collectd DPDK plugin is using; - -2. L2 statistics are only supported; - -3. The plugin has been tested on Intel NIC’s only. - -**DPDK stats known issues:** - -* DPDK port visibility - - When network port controlled by Linux is bound to DPDK driver, the port - will not be available in the OS. It affects the SNMP write plugin as those - ports will not be present in standard IF-MIB. Thus, additional work is - required to be done to support DPDK ports and statistics. - -**DPDK telemetry plugin** - -Please refer to https://wiki.anuket.io/display/HOME/DPDK+Telemetry+Plugin Hugepages Plugin ^^^^^^^^^^^^^^^^^ diff --git a/docs/release/userguide/installguide.docker.rst b/docs/release/userguide/installguide.docker.rst index 38d467a4..9141eef6 100644 --- a/docs/release/userguide/installguide.docker.rst +++ b/docs/release/userguide/installguide.docker.rst @@ -12,14 +12,14 @@ Anuket Barometer Docker Install Guide :local: The intention of this user guide is to outline how to install and test the Barometer project's -docker images. The `OPNFV docker hub <https://hub.docker.com/u/opnfv/?page=1>`_ contains 5 docker +docker images. The `Anuket docker hub <https://hub.docker.com/u/anuket/>`_ contains 5 docker images from the Barometer project: - 1. `Collectd docker image <https://hub.docker.com/r/opnfv/barometer-collectd/>`_ - 2. `Influxdb docker image <https://hub.docker.com/r/opnfv/barometer-influxdb/>`_ - 3. `Grafana docker image <https://hub.docker.com/r/opnfv/barometer-grafana/>`_ - 4. `Kafka docker image <https://hub.docker.com/r/opnfv/barometer-kafka/>`_ - 5. `VES application docker image <https://hub.docker.com/r/opnfv/barometer-ves/>`_ + 1. `Collectd docker image <https://hub.docker.com/r/anuket/barometer-collectd/>`_ + 2. `Influxdb docker image <https://hub.docker.com/r/anuket/barometer-influxdb/>`_ + 3. `Grafana docker image <https://hub.docker.com/r/anuket/barometer-grafana/>`_ + 4. `Kafka docker image <https://hub.docker.com/r/anuket/barometer-kafka>`_ + 5. `VES application docker image <https://hub.docker.com/r/anuket/barometer-ves/>`_ For description of images please see section `Barometer Docker Images Description`_ @@ -305,11 +305,11 @@ are stored in subdirectories of `docker/` directory Download the collectd docker image ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you wish to use a pre-built barometer image, you can pull the barometer -image from `dockerhub <https://hub.docker.com/r/opnfv/barometer-collectd/>`_ +image from `dockerhub <https://hub.docker.com/r/anuket/barometer-collectd/>`_ .. code:: bash - $ docker pull opnfv/barometer-collectd + $ docker pull anuket/barometer-collectd Build stable collectd container ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -317,7 +317,7 @@ Build stable collectd container .. code:: bash $ cd <BAROMETER_REPO_DIR>/docker/barometer-collectd - $ sudo docker build -t opnfv/barometer-collectd --build-arg http_proxy=`echo $http_proxy` \ + $ sudo docker build -t anuket/barometer-collectd --build-arg http_proxy=`echo $http_proxy` \ --build-arg https_proxy=`echo $https_proxy` --network=host -f Dockerfile . .. note:: @@ -330,12 +330,12 @@ Check the docker images: $ sudo docker images -Output should contain a barometer-collectd image: +Output should contain a ``barometer-collectd`` image: .. code:: REPOSITORY TAG IMAGE ID CREATED SIZE - opnfv/barometer-collectd latest 05f2a3edd96b 3 hours ago 1.2GB + anuket/barometer-collectd latest 39f5e0972178 2 months ago 1.28GB centos 7 196e0ce0c9fb 4 weeks ago 197MB centos latest 196e0ce0c9fb 4 weeks ago 197MB hello-world latest 05a3bd381fc2 4 weeks ago 1.84kB @@ -352,7 +352,7 @@ Build barometer-collectd-latest container .. code:: bash $ cd <BAROMETER_REPO_DIR> - $ sudo docker build -t opnfv/barometer-collectd-latest \ + $ sudo docker build -t anuket/barometer-collectd-latest \ --build-arg http_proxy=`echo $http_proxy` \ --build-arg https_proxy=`echo $https_proxy` --network=host -f \ docker/barometer-collectd-latest/Dockerfile . @@ -374,7 +374,7 @@ COLLECTD_PULL_REQUESTS should be a comma-delimited string of pull request IDs. .. code:: bash $ cd <BAROMETER_REPO_DIR> - $ sudo docker build -t opnfv/barometer-collectd-experimental \ + $ sudo docker build -t anuket/barometer-collectd-experimental \ --build-arg http_proxy=`echo $http_proxy` \ --build-arg https_proxy=`echo $https_proxy` \ --build-arg COLLECTD_PULL_REQUESTS=1234,5678 \ @@ -397,7 +397,7 @@ proposed patches to collectd. .. code:: bash $ cd <BAROMETER_REPO_DIR> - $ sudo docker build -t opnfv/barometer-collectd-6 \ + $ sudo docker build -t anuket/barometer-collectd-6 \ --build-arg COLLECTD_FLAVOR=collectd-6 \ --build-arg COLLECTD_PULL_REQUESTS=1234,5678 \ --network=host -f docker/barometer-collectd-experimental/Dockerfile . @@ -431,7 +431,7 @@ Run the collectd stable docker image $ sudo docker run -ti --net=host -v \ `pwd`/src/collectd/collectd_sample_configs:/opt/collectd/etc/collectd.conf.d \ -v /var/run:/var/run -v /tmp:/tmp -v /sys/fs/resctrl:/sys/fs/resctrl \ - --privileged opnfv/barometer-collectd + --privileged anuket/barometer-collectd .. note:: The docker collectd image contains configuration for all the collectd @@ -450,8 +450,7 @@ Run the collectd stable docker image files should be removed from shared configuration directory (`<BAROMETER_REPO_DIR>/src/collectd/collectd_sample_configs/`) prior to starting barometer-collectd container. By example: in case of missing - `DPDK` functionality on the host, `dpdkstat.conf` and `dpdkevents.conf` - should be removed. + `DPDK` functionality on the host, `dpdk_telemetry.conf` should be removed. Sample configurations can be found at: https://github.com/opnfv/barometer/tree/master/src/collectd/collectd_sample_configs @@ -492,7 +491,7 @@ collectd plugins requiring different configuration files) $ sudo docker run -ti --net=host -v \ `pwd`/src/collectd/collectd_sample_configs-latest:/opt/collectd/etc/collectd.conf.d \ -v /var/run:/var/run -v /tmp:/tmp -v /sys/fs/resctrl:/sys/fs/resctrl \ - --privileged opnfv/barometer-collectd-latest + --privileged anuket/barometer-collectd-latest .. note:: Barometer collectd docker images are sharing some directories with host @@ -534,7 +533,7 @@ collectd container) $ sudo docker run -ti --net=host -v \ `pwd`/src/collectd/collectd_sample_configs-latest:/opt/collectd/etc/collectd.conf.d \ -v /var/run:/var/run -v /tmp:/tmp -v /sys/fs/resctrl:/sys/fs/resctrl --privileged \ - opnfv/barometer-collectd-experimental + anuket/barometer-collectd-experimental .. note:: The Resource Control file system (/sys/fs/resctrl) can be bound from host to @@ -572,7 +571,7 @@ volume folder been mounted. Appropriate example are given in section `Run the Gr Download the InfluxDB and Grafana docker images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you wish to use pre-built barometer project's influxdb and grafana images, you can pull the -images from https://hub.docker.com/r/opnfv/barometer-influxdb/ and https://hub.docker.com/r/opnfv/barometer-grafana/ +images from https://hub.docker.com/r/anuket/barometer-influxdb/ and https://hub.docker.com/r/anuket/barometer-grafana/ .. note:: If your preference is to build images locally please see sections `Build InfluxDB Docker Image`_ and @@ -580,8 +579,8 @@ images from https://hub.docker.com/r/opnfv/barometer-influxdb/ and https://hub.d .. code:: bash - $ docker pull opnfv/barometer-influxdb - $ docker pull opnfv/barometer-grafana + $ docker pull anuket/barometer-influxdb + $ docker pull anuket/barometer-grafana .. note:: If you have pulled the pre-built barometer-influxdb and barometer-grafana images there is no @@ -597,7 +596,7 @@ Build influxdb image from Dockerfile .. code:: bash $ cd barometer/docker/barometer-influxdb - $ sudo docker build -t opnfv/barometer-influxdb --build-arg http_proxy=`echo $http_proxy` \ + $ sudo docker build -t anuket/barometer-influxdb --build-arg http_proxy=`echo $http_proxy` \ --build-arg https_proxy=`echo $https_proxy` --network=host -f Dockerfile . .. note:: @@ -615,7 +614,7 @@ Output should contain an influxdb image: .. code:: REPOSITORY TAG IMAGE ID CREATED SIZE - opnfv/barometer-influxdb latest 1e4623a59fe5 3 days ago 191MB + anuket/barometer-influxdb latest c5a09a117067 2 months ago 191MB Build Grafana docker image ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -625,7 +624,7 @@ Build Grafana image from Dockerfile .. code:: bash $ cd barometer/docker/barometer-grafana - $ sudo docker build -t opnfv/barometer-grafana --build-arg http_proxy=`echo $http_proxy` \ + $ sudo docker build -t anuket/barometer-grafana --build-arg http_proxy=`echo $http_proxy` \ --build-arg https_proxy=`echo $https_proxy` -f Dockerfile . .. note:: @@ -643,7 +642,7 @@ Output should contain an influxdb image: .. code:: REPOSITORY TAG IMAGE ID CREATED SIZE - opnfv/barometer-grafana latest 05f2a3edd96b 3 hours ago 1.2GB + anuket/barometer-grafana latest 3724ab87f0b1 2 months ago 284MB Run the Influxdb and Grafana Images ----------------------------------- @@ -653,7 +652,7 @@ Run the InfluxDB docker image .. code:: bash $ sudo docker run -tid -v /var/lib/influxdb:/var/lib/influxdb --net=host\ - --name bar-influxdb opnfv/barometer-influxdb + --name bar-influxdb anuket/barometer-influxdb Check your docker image is running @@ -714,7 +713,7 @@ changing output location is required: $ cd <BAROMETER_REPO_DIR> $ sudo docker run -ti --name bar-collectd --net=host -v \ `pwd`/src/collectd/collectd_sample_configs:/opt/collectd/etc/collectd.conf.d \ - -v /var/run:/var/run -v /tmp:/tmp --privileged opnfv/barometer-collectd + -v /var/run:/var/run -v /tmp:/tmp --privileged anuket/barometer-collectd Now collectd container will be sending data to InfluxDB container located on remote Host pointed by IP configured in step 3. @@ -729,7 +728,7 @@ Connecting to an influxdb instance running on local system and adding own custom $ cd <BAROMETER_REPO_DIR> $ sudo docker run -tid -v /var/lib/grafana:/var/lib/grafana \ -v ${PWD}/docker/barometer-grafana/dashboards:/opt/grafana/dashboards \ - --name bar-grafana --net=host opnfv/barometer-grafana + --name bar-grafana --net=host anuket/barometer-grafana Connecting to an influxdb instance running on remote system with hostname of someserver and IP address of 192.168.121.111 @@ -738,7 +737,7 @@ of 192.168.121.111 $ sudo docker run -tid -v /var/lib/grafana:/var/lib/grafana --net=host -e \ influxdb_host=someserver --add-host someserver:192.168.121.111 --name \ - bar-grafana opnfv/barometer-grafana + bar-grafana anuket/barometer-grafana Check your docker image is running @@ -786,7 +785,7 @@ Download VES and Kafka docker images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you wish to use pre-built barometer project's VES and kafka images, you can pull the -images from https://hub.docker.com/r/opnfv/barometer-ves/ and https://hub.docker.com/r/opnfv/barometer-kafka/ +images from https://hub.docker.com/r/anuket/barometer-ves/ and https://hub.docker.com/r/anuket/barometer-kafka/ .. note:: If your preference is to build images locally please see sections `Build Kafka Docker Image`_ and @@ -794,8 +793,8 @@ images from https://hub.docker.com/r/opnfv/barometer-ves/ and https://hub.docke .. code:: bash - $ docker pull opnfv/barometer-kafka - $ docker pull opnfv/barometer-ves + $ docker pull anuket/barometer-kafka + $ docker pull anuket/barometer-ves .. note:: If you have pulled the pre-built images there is no requirement to complete steps outlined @@ -810,7 +809,7 @@ Build Kafka docker image: .. code:: bash $ cd barometer/docker/barometer-kafka - $ sudo docker build -t opnfv/barometer-kafka --build-arg http_proxy=`echo $http_proxy` \ + $ sudo docker build -t anuket/barometer-kafka --build-arg http_proxy=`echo $http_proxy` \ --build-arg https_proxy=`echo $https_proxy` -f Dockerfile . .. note:: @@ -828,7 +827,7 @@ Output should contain a barometer image: .. code:: REPOSITORY TAG IMAGE ID CREATED SIZE - opnfv/barometer-kafka latest 05f2a3edd96b 3 hours ago 1.2GB + anuket/barometer-kafka latest 75a0860b8d6e 2 months ago 902MB Build VES docker image ^^^^^^^^^^^^^^^^^^^^^^ @@ -838,7 +837,7 @@ Build VES application docker image: .. code:: bash $ cd barometer/docker/barometer-ves - $ sudo docker build -t opnfv/barometer-ves --build-arg http_proxy=`echo $http_proxy` \ + $ sudo docker build -t anuket/barometer-ves --build-arg http_proxy=`echo $http_proxy` \ --build-arg https_proxy=`echo $https_proxy` -f Dockerfile . .. note:: @@ -856,7 +855,7 @@ Output should contain a barometer image: .. code:: REPOSITORY TAG IMAGE ID CREATED SIZE - opnfv/barometer-ves latest 05f2a3edd96b 3 hours ago 1.2GB + anuket/barometer-ves latest 36a4a953e1b4 2 months ago 723MB Run Kafka docker image ^^^^^^^^^^^^^^^^^^^^^^ @@ -881,7 +880,7 @@ Run kafka docker image which connects with a zookeeper instance running on same .. code:: bash - $ sudo docker run -tid --net=host -p 9092:9092 opnfv/barometer-kafka + $ sudo docker run -tid --net=host -p 9092:9092 anuket/barometer-kafka Run kafka docker image which connects with a zookeeper instance running on a node with IP address of @@ -890,7 +889,7 @@ Run kafka docker image which connects with a zookeeper instance running on a nod .. code:: bash $ sudo docker run -tid --net=host -p 9092:9092 --env broker_id=1 --env zookeeper_node=zookeeper --add-host \ - zookeeper:192.168.121.111 opnfv/barometer-kafka + zookeeper:192.168.121.111 anuket/barometer-kafka Run VES Application docker image ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -907,13 +906,13 @@ Run VES docker image with default configuration .. code:: bash - $ sudo docker run -tid --net=host opnfv/barometer-ves + $ sudo docker run -tid --net=host anuket/barometer-ves Run VES docker image with guest.yaml files from barometer/3rd_party/collectd-ves-app/ves_app/yaml/ .. code:: bash - $ sudo docker run -tid --net=host opnfv/barometer-ves guest.yaml + $ sudo docker run -tid --net=host anuket/barometer-ves guest.yaml Run VES docker image with using custom config and yaml files. In example below yaml/ folder cotains @@ -922,7 +921,7 @@ file named custom.yaml .. code:: bash $ sudo docker run -tid --net=host -v ${PWD}/custom.config:/opt/ves/config/ves_app_config.conf \ - -v ${PWD}/yaml/:/opt/ves/yaml/ opnfv/barometer-ves custom.yaml + -v ${PWD}/yaml/:/opt/ves/yaml/ anuket/barometer-ves custom.yaml Run VES Test Collector application ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/release/userguide/installguide.oneclick.rst b/docs/release/userguide/installguide.oneclick.rst index d236b5b1..78203a12 100644 --- a/docs/release/userguide/installguide.oneclick.rst +++ b/docs/release/userguide/installguide.oneclick.rst @@ -231,6 +231,36 @@ Verify that key is added and password is not required to connect. example. For multinode installation keys need to be copied for each node: [collectd_hostname], [influxdb_hostname] etc. +Build the Collectd containers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is an optional step, if you do not wish to build the containers locally, please continue to `Download and run Collectd+Influxdb+Grafana containers`_. +This step will build the container images locally, allowing for testing of new changes to collectd. +This is particularly useful for the ``experimental`` flavour for testing PRs, and for building a ``collectd-6`` container. + +To run the playbook and build the containers, run:: + sudo ansible-playbook docker/ansible/collectd_build.yml + +By default, all contaienrs will be built. +Since this can take a while, it is recommended that you choose a flavor to build using tags:: + + sudo ansible-playbook docker/ansible/collectd_build.yml --tags='collectd-6,latest' + +The available tags are: + +* *stable* builds the ``barometer-collectd`` image +* *latest* builds the ``barometer-collectd-latest`` image +* *experimental* builds the ``barometer-collectd-experimental`` container, with optional PRs +* *collectd-6* builds the ``baromter-collectd-6`` container, with optional PR(s) + +* *flask_test* builds a small webapp that displays the metrics sent via the write_http plugin + +.. note:: + The flask_test tag must be explicitly enabled. + This can be done either through the ``--tags='flask_test'`` (to build just + this container) or with ``--tags=all`` to build this and all the other + containers as well. + Download and run Collectd+Influxdb+Grafana containers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -247,10 +277,10 @@ Check the three containers are running, the output of ``docker ps`` should be si .. code:: bash $ sudo docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - a033aeea180d opnfv/barometer-grafana "/run.sh" 9 days ago Up 7 minutes bar-grafana - 1bca2e4562ab opnfv/barometer-influxdb "/entrypoint.sh in..." 9 days ago Up 7 minutes bar-influxdb - daeeb68ad1d5 opnfv/barometer-collectd "/run_collectd.sh ..." 9 days ago Up 7 minutes bar-collectd + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 4c2143fb6bbd anuket/barometer-grafana "/run.sh" 59 minutes ago Up 4 minutes bar-grafana + 5e356cb1cb04 anuket/barometer-influxdb "/entrypoint.sh infl…" 59 minutes ago Up 4 minutes bar-influxdb + 2ddac8db21e2 anuket/barometer-collectd "/run_collectd.sh" About an hour ago Up 4 minutes bar-collectd To make some changes when a container is running run: @@ -284,10 +314,10 @@ Check the containers are running, the output of ``docker ps`` should be similar $ sudo docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 29035be2dab5 zookeeper:3.4.11 "/docker-entrypoint._" 7 minutes ago Up 7 minutes bar-zookeeper - eb8bba3c0b76 opnfv/barometer-ves "./start_ves_app.s..." 6 minutes ago Up 6 minutes bar-ves - 86702a96a68c opnfv/barometer-kafka "/src/start_kafka.sh" 6 minutes ago Up 6 minutes bar-kafka - daeeb68ad1d5 opnfv/barometer-collectd "/run_collectd.sh ..." 6 minutes ago Up 6 minutes bar-collectd + d041d8fff849 zookeeper:3.4.11 "/docker-entrypoint.…" 2 minutes ago Up 2 minutes bar-zookeeper + da67b81274bc anuket/barometer-ves "./start_ves_app.sh …" 2 minutes ago Up 2 minutes bar-ves + 2c25e0c79f93 anuket/barometer-kafka "/src/start_kafka.sh" 2 minutes ago Up 2 minutes bar-kafka + b161260c90ed anuket/barometer-collectd "/run_collectd.sh" 2 minutes ago Up 2 minutes bar-collectd To make some changes when a container is running run: @@ -319,8 +349,9 @@ the following plugins: ``ethstat``, ``ipc``, ``irq``, ``load``, ``memory``, ``numa``, ``processes``, ``swap``, ``turbostat``, ``uuid``, ``uptime``, ``exec``, ``hugepages``, ``intel_pmu``, ``ipmi``, ``write_kafka``, ``logfile``, - ``mcelog``, ``network``, ``intel_rdt``, ``rrdtool``, ``snmp_agent``, - ``syslog``, ``virt``, ``ovs_stats``, ``ovs_events``, ``dpdk_telemetry`` + ``logparser``, ``mcelog``, ``network``, ``intel_rdt``, ``rrdtool``, + ``snmp_agent``, ``syslog``, ``virt``, ``ovs_stats``, ``ovs_events``, + ``dpdk_telemetry``. .. note:: Some of the plugins are loaded depending on specific system requirements and can be omitted if @@ -330,15 +361,6 @@ the following plugins: For instructions on how to disable certain plugins see the `List and description of tags used in ansible scripts`_ section. -.. note:: - The ``dpdkstat`` and ``dpdkevents`` plugins are disabled by default (in - favour of the ``dpdk_telemetry`` plugin) and need to be explicitly enabled - in order to use them: - - .. code:: bash - - $ sudo ansible-playbook -i default.inv collectd_service.yml --tags "all,dpdkstats,dpdkevents" - List and description of tags used in ansible scripts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -382,7 +404,7 @@ List of available tags: ``contextswitch``, ``cpu``, ``cpufreq``, ``df``, ``disk,`` ``ethstat``, ``ipc``, ``irq``, ``load``, ``memory``, ``numa``, ``processes``, ``swap``, ``turbostat``, ``uptime``, ``exec``, ``hugepages``, ``ipmi``, ``kafka``, - ``logfile``, ``mcelog``, ``network``, ``pmu``, ``rdt``, ``rrdtool``, - ``snmp``, ``syslog``, ``unixsock``, ``virt``, ``ovs_stats``, ``ovs_events``, - ``uuid``, ``dpdkevents``, ``dpdkstat``, ``dpdk_telemetry``. + ``logfile``, ``logparser``, ``mcelog``, ``network``, ``pmu``, ``rdt``, + ``rrdtool``, ``snmp``, ``syslog``, ``unixsock``, ``virt``, ``ovs_stats``, + ``ovs_events``, ``uuid``, ``dpdk_telemetry``. diff --git a/docs/testing/index.rst b/docs/testing/index.rst index 392b39f4..f763ca64 100644 --- a/docs/testing/index.rst +++ b/docs/testing/index.rst @@ -1 +1,79 @@ -.. To be decided +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) Anuket and others + +============================== +Anuket Barometer testing guide +============================== + +This document will describe how to use different tests in this repo. + +There are a number of tools and scripts in Barometer that can be used for testing, whether that is during development, building, code reviews, or run regularly in CI. +Some of the tests are automated, and cover building collectd, others cover particular plugins. + +.. TODO: This guide should also include how to manually verify that collectd plugins are working as expected. + +.. TODO: There might be some troubleshooting guide in here too. + +Porting collectd to version 6 +============================= + +Thre is an ansible playbook for building and running collectd 5 and 6 together to compare the collected metrics. +This is intended to help test porting from collectd 5 to 6, and confirm equivalency across the versions. + +The playbook will:: + + * build collectd-6, collectd-latest and flask app containers + * generate a set of collectd configs + * launch the collectd-6, collectd-latest with the generated configs + * run the flask app which has a http server that receives metrics from + collectd v5 and collectd v6 + * display the received metrics from both versions of collectd + Collectd v5 shows PUTVAL + Collectd v6 shows PUTMETRIC + +To run this comparison, use the following command:: + + $ cd docker/ansible/ + $ sudo ansible-playbook -i default.inv collectd6_test.yml + +The playbook takes the following parameters: + + * PR (optional) + The PRID for an upstream collectd pull request that will be + passed to the collectd 6 container build + + * plugin (optional) + The name of the plugin that is bneing ported + This will filter the received metrics to show the value passed. + +To run the playbook with these configs, pass the extra var to ansible:: + + sudo ansible-playbook -i default.inv -e PR=<PR_ID> -e plugin=<plugin_name> collectd6_test.yml + +The metrics can then be viewed by inspecting the container logs or attaching to the container to view the output:: + + $ docker attach <webserver-container> + $ #OR + $ docker logs <webserver-container> + +Metrics from collectd 5 will appear preceeded with ``PUTVAL``, and metrics from collectd 6 will appear preceeded by ``PUTMETRIC``. + +:: + + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-2048Kb/vmpage_number-free interval=10.000 1629466502.664:0 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-2048Kb/vmpage_number-used interval=10.000 1629466502.664:0 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-1048576Kb/vmpage_number-free interval=10.000 1629466502.664:0 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-1048576Kb/vmpage_number-used interval=10.000 1629466502.664:0 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-2048Kb/vmpage_number-free interval=10.000 1629466502.664:0 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-2048Kb/vmpage_number-used interval=10.000 1629466502.664:0 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-1048576Kb/vmpage_number-used interval=10.000 1629466502.665:0 + PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-1048576Kb/vmpage_number-free interval=10.000 1629466502.665:0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.807 interval=10.000 label:hugepages="mm-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.807 interval=10.000 label:hugepages="mm-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="mm-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="node0-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="node0-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.809 interval=10.000 label:hugepages="node0-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.809 interval=10.000 label:hugepages="node0-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0 + PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="mm-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0 diff --git a/src/collectd/collectd_apply_pull_request.sh b/src/collectd/collectd_apply_pull_request.sh index 35af2981..403d78dd 100755 --- a/src/collectd/collectd_apply_pull_request.sh +++ b/src/collectd/collectd_apply_pull_request.sh @@ -35,8 +35,16 @@ IFS=', ' read -a PULL_REQUESTS <<< "$COLLECTD_PULL_REQUESTS" git config user.email "barometer-experimental@container" git config user.name "BarometerExperimental" -for PR_ID in "${PULL_REQUESTS[@]}" -do - echo "Applying pull request $PR_ID" - git pull --rebase origin pull/$PR_ID/head -done +# If there's a single PR listed, just check it out +if [ "${#PULL_REQUESTS[@]}" -eq "1" ]; +then + echo "Checking out pull request $COLLECTD_PULL_REQUESTS" + git fetch origin pull/$COLLECTD_PULL_REQUESTS/head && git checkout FETCH_HEAD +else +# if there are multiple PRs, rebase them on top of the checked out branch + for PR_ID in "${PULL_REQUESTS[@]}" + do + echo "Applying pull request $PR_ID" + git pull --rebase origin pull/$PR_ID/head + done +fi diff --git a/src/collectd/collectd_sample_configs-latest/dpdkevents.conf b/src/collectd/collectd_sample_configs-latest/dpdkevents.conf deleted file mode 100644 index fdb6f3db..00000000 --- a/src/collectd/collectd_sample_configs-latest/dpdkevents.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2017 OPNFV -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -<LoadPlugin dpdkevents> - Interval 1 -</LoadPlugin> - -<Plugin "dpdkevents"> - <EAL> -# Coremask "0x1" -# MemoryChannels "4" -# FilePrefix "rte" - </EAL> - <Event "link_status"> - SendEventsOnUpdate false - EnabledPortMask 0xffff - SendNotification true - </Event> - <Event "keep_alive"> - SendEventsOnUpdate false - LCoreMask "0xf" - KeepAliveShmName "/dpdk_keepalive_shm_name" - SendNotification true - </Event> -</Plugin> diff --git a/src/collectd/collectd_sample_configs/dpdkevents.conf b/src/collectd/collectd_sample_configs/dpdkevents.conf deleted file mode 100644 index fdb6f3db..00000000 --- a/src/collectd/collectd_sample_configs/dpdkevents.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2017 OPNFV -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -<LoadPlugin dpdkevents> - Interval 1 -</LoadPlugin> - -<Plugin "dpdkevents"> - <EAL> -# Coremask "0x1" -# MemoryChannels "4" -# FilePrefix "rte" - </EAL> - <Event "link_status"> - SendEventsOnUpdate false - EnabledPortMask 0xffff - SendNotification true - </Event> - <Event "keep_alive"> - SendEventsOnUpdate false - LCoreMask "0xf" - KeepAliveShmName "/dpdk_keepalive_shm_name" - SendNotification true - </Event> -</Plugin> diff --git a/systems/centos/8/build_base_machine.sh b/systems/centos/8/build_base_machine.sh index 1a590841..bb3caa12 100755 --- a/systems/centos/8/build_base_machine.sh +++ b/systems/centos/8/build_base_machine.sh @@ -52,6 +52,7 @@ pkg-config git-core sudo rpm-build +which libcap-devel xfsprogs-devel iptables-devel @@ -22,3 +22,8 @@ commands = sphinx-build -b linkcheck -d {envtmpdir}/doctrees ./docs/ {toxinidir} basepython=python3 deps = -rdocs/requirements.txt commands = reno --rel-notes-dir docs/release/release-notes/ report + +[testenv:docs-create-note] +basepython = python3 +deps = -rdocs/requirements.txt +commands = reno --rel-notes-dir docs/release/release-notes/ new {posargs} |