diff options
| -rw-r--r-- | ci/barometer-build.sh | 26 | ||||
| -rw-r--r-- | ci/barometer-upload-artifact.sh | 74 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/defaults/main.yml | 60 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/tasks/default_read_import.yml | 22 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/tasks/default_read_plugins.yml | 34 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/tasks/main.yml | 31 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/templates/default_read_plugins.conf.j2 | 4 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/vars/main.yml | 53 | ||||
| -rw-r--r-- | docker/ansible/roles/run_collectd/tasks/main.yml | 6 | ||||
| -rw-r--r-- | docs/release/release-notes/index.rst | 1 | ||||
| -rw-r--r-- | docs/release/release-notes/kali-release-notes.rst | 40 |
11 files changed, 261 insertions, 90 deletions
diff --git a/ci/barometer-build.sh b/ci/barometer-build.sh new file mode 100644 index 00000000..04bbd263 --- /dev/null +++ b/ci/barometer-build.sh @@ -0,0 +1,26 @@ +# This script is used by the barometer-daily CI job in gitlab. +# It builds and packages collectd as an RPM +# After this script is run, the baroemter-daily job runs the +# barometer-upload-artifact.sh script. +set -x + +OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +OPNFV_ARTIFACT_URL="$GS_URL/$OPNFV_ARTIFACT_VERSION/" + +# log info to console +echo "Starting the build of Barometer RPMs" +echo "------------------------------------" +echo + +cd ci +./install_dependencies.sh +./build_rpm.sh +cp utility/rpms_list $WORKSPACE +cd $WORKSPACE + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_ARTIFACT_URL=$OPNFV_ARTIFACT_URL" +) > $WORKSPACE/opnfv.properties + diff --git a/ci/barometer-upload-artifact.sh b/ci/barometer-upload-artifact.sh new file mode 100644 index 00000000..f05dc2af --- /dev/null +++ b/ci/barometer-upload-artifact.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -o nounset +set -o pipefail + +RPM_LIST=$WORKSPACE/rpms_list +RPM_WORKDIR=$WORKSPACE/rpmbuild +RPM_DIR=$RPM_WORKDIR/RPMS/x86_64/ +cd $WORKSPACE/ + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +# Check if all the appropriate RPMs were generated +echo "Checking if all the Barometer RPMs were created" +echo "-----------------------------------------------" +echo + +if [ -d $RPM_DIR ] +then + ls $RPM_DIR > list_of_gen_pack +else + echo "Can't access folder $RPM_DIR with rpm packages" + echo "Barometer nightly build FAILED" + exit 1 +fi + +for PACKAGENAME in `cat $RPM_LIST` +do + if ! grep -q $PACKAGENAME list_of_gen_pack + then + echo "$PACKAGENAME is missing" + echo "Barometer nightly build FAILED" + exit 2 + fi +done + +#remove the file you no longer need. +rm list_of_gen_pack + +echo "Uploading the barometer RPMs to artifacts.opnfv.org" +echo "---------------------------------------------------" +echo + +gsutil -m cp -r $RPM_DIR/* gs://$OPNFV_ARTIFACT_URL > $WORKSPACE/gsutil.log 2>&1 + +# Check if the RPMs were pushed +gsutil ls gs://$OPNFV_ARTIFACT_URL > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading barometer RPMs to gs://$OPNFV_ARTIFACT_URL!" + echo "Check log $WORKSPACE/gsutil.log on the appropriate build server" + exit 1 +else + # upload property files only if build is successful + gsutil cp $WORKSPACE/opnfv.properties gs://$OPNFV_ARTIFACT_URL/opnfv.properties > gsutil.properties.log 2>&1 + gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +fi + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$OPNFV_ARTIFACT_URL/*.rpm > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$OPNFV_ARTIFACT_URL/opnfv.properties > /dev/null 2>&1 + +echo +echo "--------------------------------------------------------" +echo "Done!" +echo "Artifact is available at $OPNFV_ARTIFACT_URL" + +#cleanup the RPM repo from the build machine. +rm -rf $RPM_WORKDIR diff --git a/docker/ansible/roles/config_files/defaults/main.yml b/docker/ansible/roles/config_files/defaults/main.yml new file mode 100644 index 00000000..c2004ae1 --- /dev/null +++ b/docker/ansible/roles/config_files/defaults/main.yml @@ -0,0 +1,60 @@ +# Copyright 2018-21 Anuket, Intel Corporation and others +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +config_file_dir: "/opt/collectd/etc/collectd.conf.d/" +csv_log_dir: "/var/lib/collectd/csv" +logfile_dir: "/var/log/" + +#global default interval +interval_value: 10 + +#influx network port +influx_network_port: 25826 + +#network plugin vars +network_port: 25826 +network_ip_addr: localhost + +#prometheus plugin vars +prometheus_port: 9103 + +#ovs event and stats plugin vars +ovs_events_interval: 1 +ovs_events_ip_addr: localhost +ovs_events_port: 6640 +ovs_stats_interval: 1 +ovs_stats_ip_addr: localhost +ovs_stats_port: 6640 + +#rdt plugin vars +rdt_interval: 1 + +#kafka plugin vars +kafka_ip_addr: localhost +kafka_port: 9092 +kafka_topic: collectd + +#logfile plugin vars +logfile_log_level: info + +#syslog plugin vars +syslog_log_level: info + +#rrd plugin vars +rrdtool_db_dir: "/var/lib/collectd/rrd" +cache_flush: 120 +write_per_sec: 50 + +#additional configuration files path +additional_configs_path: "" diff --git a/docker/ansible/roles/config_files/tasks/default_read_import.yml b/docker/ansible/roles/config_files/tasks/default_read_import.yml index d70919b3..3c7d8766 100644 --- a/docker/ansible/roles/config_files/tasks/default_read_import.yml +++ b/docker/ansible/roles/config_files/tasks/default_read_import.yml @@ -13,12 +13,20 @@ # limitations under the License. --- +- name: copy default template to system + template: + src: default_read_plugins.conf.j2 + dest: "{{ config_file_dir }}/default_read_plugins.conf" + tags: + - always + - name: enable contextswitch plugin replace: path: "{{ config_file_dir }}/default_read_plugins.conf" regexp: '(\s+)#LoadPlugin contextswitch(\s+.*)?$' replace: '\1LoadPlugin contextswitch\2' tags: + - en_default_all - contextswitch - name: enable cpu plugin @@ -27,6 +35,7 @@ regexp: '(\s+)#LoadPlugin cpu(\s+.*)?$' replace: '\1LoadPlugin cpu\2' tags: + - en_default_all - cpu - name: enable cpufreq plugin @@ -35,6 +44,7 @@ regexp: '(\s+)#LoadPlugin cpufreq(\s+.*)?$' replace: '\1LoadPlugin cpufreq\2' tags: + - en_default_all - cpufreq - name: enable df plugin @@ -43,6 +53,7 @@ regexp: '(\s+)#LoadPlugin df(\s+.*)?$' replace: '\1LoadPlugin df\2' tags: + - en_default_all - df - name: enable disk plugin @@ -51,6 +62,7 @@ regexp: '(\s+)#LoadPlugin disk(\s+.*)?$' replace: '\1LoadPlugin disk\2' tags: + - en_default_all - disk - name: enable ethstat plugin @@ -59,6 +71,7 @@ regexp: '(\s+)#LoadPlugin ethstat(\s+.*)?$' replace: '\1LoadPlugin ethstat\2' tags: + - en_default_all - ethstat - name: enable ipc plugin @@ -67,6 +80,7 @@ regexp: '(\s+)#LoadPlugin ipc(\s+.*)?$' replace: '\1LoadPlugin ipc\2' tags: + - en_default_all - ipc - name: enable irq plugin @@ -75,6 +89,7 @@ regexp: '(\s+)#LoadPlugin irq(\s+.*)?$' replace: '\1LoadPlugin irq\2' tags: + - en_default_all - irq - name: enable load plugin @@ -83,6 +98,7 @@ regexp: '(\s+)#LoadPlugin load(\s+.*)?$' replace: '\1LoadPlugin load\2' tags: + - en_default_all - load - name: enable memory plugin @@ -91,6 +107,7 @@ regexp: '(\s+)#LoadPlugin memory(\s+.*)?$' replace: '\1LoadPlugin memory\2' tags: + - en_default_all - memory - name: enable numa plugin @@ -99,6 +116,7 @@ regexp: '(\s+)#LoadPlugin numa(\s+.*)?$' replace: '\1LoadPlugin numa\2' tags: + - en_default_all - numa - name: enable processes plugin @@ -107,6 +125,7 @@ regexp: '(\s+)#LoadPlugin processes(\s+.*)?$' replace: '\1LoadPlugin processes\2' tags: + - en_default_all - processes - name: enable swap plugin @@ -115,6 +134,7 @@ regexp: '(\s+)#LoadPlugin swap(\s+.*)?$' replace: '\1LoadPlugin swap\2' tags: + - en_default_all - swap - name: enable turbostat plugin @@ -123,6 +143,7 @@ regexp: '(\s+)#LoadPlugin turbostat(\s+.*)?$' replace: '\1LoadPlugin turbostat\2' tags: + - en_default_all - turbostat - name: enable uptime plugin @@ -131,4 +152,5 @@ regexp: '(\s+)#LoadPlugin uptime(\s+.*)?$' replace: '\1LoadPlugin uptime\2' tags: + - en_default_all - uptime diff --git a/docker/ansible/roles/config_files/tasks/default_read_plugins.yml b/docker/ansible/roles/config_files/tasks/default_read_plugins.yml deleted file mode 100644 index 95b942c2..00000000 --- a/docker/ansible/roles/config_files/tasks/default_read_plugins.yml +++ /dev/null @@ -1,34 +0,0 @@ -#Copyright 2018 OPNFV and Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: copy default template to system - template: - src: default_read_plugins.conf.j2 - dest: "{{ config_file_dir }}/default_read_plugins.conf" - tags: - - always - -- name: enable all default plugins - replace: - path: "{{ config_file_dir }}/default_read_plugins.conf" - regexp: '(\s+)#LoadPlugin(\s+.*)?$' - replace: '\1LoadPlugin\2' - register: en_default_all - tags: - - en_default_all - -- name: include per default plugin tasks - include: default_read_import.yml - when: en_default_all is not defined
\ No newline at end of file diff --git a/docker/ansible/roles/config_files/tasks/main.yml b/docker/ansible/roles/config_files/tasks/main.yml index 3221242d..4a25e4eb 100644 --- a/docker/ansible/roles/config_files/tasks/main.yml +++ b/docker/ansible/roles/config_files/tasks/main.yml @@ -1,4 +1,5 @@ -# Copyright 2018-2019 Intel Corporation and OPNFV. All rights reserved. +# Copyright 2018-2021 Intel Corporation, Anuket and others. +# All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,7 +38,7 @@ import_tasks: csv.yml - name: enable default plugins - import_tasks: default_read_plugins.yml + import_tasks: default_read_import.yml - name: enable dpdk plugins import_tasks: dpdk.yml @@ -91,5 +92,31 @@ - name: enable uuid plugin include: uuid.yml +- name: configure plugins + include_role: + name: collectd_config + vars: + collectd_conf_output_dir: /tmp/collectd.conf.d + tags: + - always + - en_default_all + +- name: "Copy the generated plugin configs" + shell: + sudo cp /tmp/collectd.conf.d/*.conf {{ config_file_dir }} && sudo cp /tmp/collectd.conf {{ config_file_dir }}/../collectd.conf + tags: + - always + - en_default_all + +- name: Update config file location + lineinfile: + path: "{{ config_file_dir }}../collectd.conf" + regexp: "^(.*)/tmp/collectd.conf.d(.*)$" + line: '\1{{ config_file_dir }}\2' + backrefs: yes + tags: + - always + - en_default_all + - name: copy additional config files include: additional_configs.yml diff --git a/docker/ansible/roles/config_files/templates/default_read_plugins.conf.j2 b/docker/ansible/roles/config_files/templates/default_read_plugins.conf.j2 index 48beb0be..4400a23b 100644 --- a/docker/ansible/roles/config_files/templates/default_read_plugins.conf.j2 +++ b/docker/ansible/roles/config_files/templates/default_read_plugins.conf.j2 @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -Hostname "{{ inventory_hostname }}" -Interval "{{ interval_value }}" +Hostname "{{ collectd_hostname }}" +Interval "{{ collectd_interval }}" #LoadPlugin contextswitch diff --git a/docker/ansible/roles/config_files/vars/main.yml b/docker/ansible/roles/config_files/vars/main.yml index f12492a3..5af4d7a0 100644 --- a/docker/ansible/roles/config_files/vars/main.yml +++ b/docker/ansible/roles/config_files/vars/main.yml @@ -1,4 +1,4 @@ -#Copyright 2018 OPNFV and Intel Corporation +# Copyright 2018-21 Anuket, Intel Corporation and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,51 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - -#directory paths -config_file_dir: "/opt/collectd/etc/collectd.conf.d/" -csv_log_dir: "/var/lib/collectd/csv" -logfile_dir: "/var/log/" - -#global default interval -interval_value: 10 - -#influx network port -influx_network_port: 25826 - -#network plugin vars -#network_port: 25826 -#network_ip_addr: localhost - -#prometheus plugin vars -prometheus_port: 9103 - -#ovs event and stats plugin vars -ovs_event_interval: 1 -ovs_event_ip_addr: localhost -ovs_event_port: 6640 -ovs_stats_interval: 1 -ovs_stats_ip_addr: localhost -ovs_stats_port: 6640 - -#rdt plugin vars -rdt_interval: 1 - -#kafka plugin vars -kafka_ip_addr: localhost -kafka_port: 9092 -kafka_topic: collectd - -#logfile plugin vars -logfile_log_level: info - -#syslog plugin vars -syslog_log_level: info - -#rrd plugin vars -rrdtool_db_dir: "/var/lib/collectd/rrd" -cache_flush: 120 -write_per_sec: 50 - -#additional configuration files path -additional_configs_path: "" +collectd_plugins: [] +collectd_interval: "{{ interval_value }}" +collectd_hostname: "{{ inventory_hostname }}" diff --git a/docker/ansible/roles/run_collectd/tasks/main.yml b/docker/ansible/roles/run_collectd/tasks/main.yml index 8fc27342..2c5d0e67 100644 --- a/docker/ansible/roles/run_collectd/tasks/main.yml +++ b/docker/ansible/roles/run_collectd/tasks/main.yml @@ -1,4 +1,4 @@ -#Copyright 2018-2019 OPNFV and Intel Corporation +# Copyright 2018-21 Anuket, Intel Corporation and others # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ set_fact: volumes_list: - - /opt/collectd/etc/collectd.conf.d/:/opt/collectd/etc/collectd.conf.d + - /opt/collectd/etc/:/opt/collectd/etc/ - /var/run:/var/run - /tmp:/tmp - /var/lib/collectd:/var/lib/collectd @@ -55,7 +55,7 @@ name: bar-collectd image: "{{ collectd_image_name }}" volumes: "{{ volumes_list }}" - command: "/run_collectd.sh" + entrypoint: "/run_collectd.sh" detach: yes state: started restart: yes diff --git a/docs/release/release-notes/index.rst b/docs/release/release-notes/index.rst index 9eb1348e..df4696a5 100644 --- a/docs/release/release-notes/index.rst +++ b/docs/release/release-notes/index.rst @@ -11,5 +11,6 @@ Anuket Barometer Release Notes .. toctree:: :maxdepth: 1 + kali-release-notes release-notes diff --git a/docs/release/release-notes/kali-release-notes.rst b/docs/release/release-notes/kali-release-notes.rst new file mode 100644 index 00000000..250becc1 --- /dev/null +++ b/docs/release/release-notes/kali-release-notes.rst @@ -0,0 +1,40 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +============================ +Barometer Kali Release Notes +============================ + +This document provides the release notes for Kali release of Barometer. + +Summary +------- +The Kali release is the first one since becoming part of Anuket, and focussed +on changes that will make testing and integrating easier. + +Details +------- +Testing and build tools were developed and updated to do the following: + +* A new reference container was added for the collectd-6.0 version, which is + under development and represents a big API change that is not backwards + compatible. This reference build should facilitate porting the plugins that + were previously developed by the Barometer project. + https://jira.anuket.io/browse/BAROMETER-184 + +* Updated to the stable version of collectd to collectd 5.12. + +* Removed duplication in the three existing containers (stable, latest and experimental). + https://jira.anuket.io/browse/BAROMETER-179 + +Some work was started but not completed in the Kali release: + +* Updating of the ansible playbooks for generating configs so that they will be + easier to maintain and extend in the future. + +* Additional testing tools for verifying plugin functionality + +References +---------- +* `Barometer Kali release plan <https://wiki.anuket.io/display/HOME/Barometer+Kali+Release+Planning>`_ +* `Kali Release on Jira <https://jira.anuket.io/projects/BAROMETER/versions/10224>`_ |