diff options
| -rw-r--r-- | .gitlab-ci.yml | 165 | ||||
| -rw-r--r-- | ci/barometer-build.sh | 26 | ||||
| -rw-r--r-- | ci/barometer-upload-artifact.sh | 74 | ||||
| -rw-r--r-- | docker/ansible/default.inv | 4 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/tasks/snmp_agent.yml | 4 | ||||
| -rw-r--r-- | docker/ansible/roles/config_files/templates/latest/snmp_agent.conf.j2 (renamed from docker/ansible/roles/config_files/templates/master/snmp_agent.conf.j2) | 0 | ||||
| -rw-r--r-- | docs/release/release-notes/index.rst | 1 | ||||
| -rw-r--r-- | docs/release/release-notes/kali-release-notes.rst | 40 | ||||
| -rwxr-xr-x | systems/centos/8/build_base_machine.sh | 16 |
9 files changed, 316 insertions, 14 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..5feacd04 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,165 @@ +--- +include: + - project: anuket/releng + file: '/gitlab-templates/RTD.gitlab-ci.yml' + - project: anuket/releng + file: '/gitlab-templates/Docker.gitlab-ci.yml' + - project: anuket/releng + file: '/gitlab-templates/GoogleStorage.gitlab-ci.yml' + +variables: + DOCKER_REGISTRY: docker.io + +.barometer-build-script: &barometer-build-script + before_script: + - ./systems/centos/8/build_base_machine.sh + script: + - pwd + - cd src + - | + echo -e "\e[0Ksection_start:`date +%s`:make_clobber\r\e[0KMake Clobber" + make clobber + echo -e "\e[0Ksection_end:`date +%s`:make_clobber\r\e[0K" + - | + echo -e "\e[0Ksection_start:`date +%s`:make\r\e[0KMake" + make + echo -e "\e[0Ksection_end:`date +%s`:make\r\e[0K" + +# Docker Builds +docker-build-collectd: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-collectd" + DOCKER_BUILDCONTEXT: "docker/barometer-collectd" + DOCKER_FILEPATH: "docker/barometer-collectd/Dockerfile" + +docker-build-collectd-latest: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-collectd-latest" + DOCKER_FILEPATH: "docker/barometer-collectd-latest/Dockerfile" + +docker-build-dma: + extends: .docker-build-and-push + # Failing since 2019 + allow_failure: true + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-dma" + DOCKER_BUILDCONTEXT: "docker/barometer-dma" + DOCKER_FILEPATH: "docker/barometer-dma/Dockerfile" + +docker-build-collectd-experimental: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-collectd-latest" + DOCKER_FILEPATH: "docker/barometer-collectd-experimental/Dockerfile" + +docker-build-grafana: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-grafana" + DOCKER_BUILDCONTEXT: "docker/barometer-grafana" + DOCKER_FILEPATH: "docker/barometer-grafana/Dockerfile" + +docker-build-influxdb: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-influxdb" + DOCKER_BUILDCONTEXT: "docker/barometer-influxdb" + DOCKER_FILEPATH: "docker/barometer-influxdb/Dockerfile" + +docker-build-kafka: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-kafka" + DOCKER_BUILDCONTEXT: "docker/barometer-kafka" + DOCKER_FILEPATH: "docker/barometer-kafka/Dockerfile" + +docker-build-ves: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-ves" + DOCKER_BUILDCONTEXT: "docker/barometer-ves" + DOCKER_FILEPATH: "docker/barometer-ves/Dockerfile" + +docker-build-snmp: + extends: .docker-build-and-push + variables: + DOCKER_IMAGE: "$DOCKER_ORGANIZATION/barometer-snmp" + DOCKER_BUILDCONTEXT: "docker/barometer-snmp" + DOCKER_FILEPATH: "docker/barometer-snmp/Dockerfile" + +# Build jobs +barometer-build: + image: centos:8 + stage: deploy + <<: *barometer-build-script + variables: + DOCKER: y + rules: + - changes: + - "docs/**" + - .gitignore + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +# Scheduled Jobs +barometer-daily: + image: centos:8 + stage: build + <<: *barometer-build-script + variables: + DOCKER: y + script: + - !reference [.gsutil-install, script] + - cd ci/ + - ./barometer-build.sh + - ./barometer-upload-artifact.sh + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $BAROMETER_SCHEDULE == "true" + +barometer-plugins-test: + image: docker:latest + stage: test + interruptible: true + # Build has been failing for awhile + allow_failure: true + needs: [] + services: + - docker:dind + before_script: + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_TOKEN" $DOCKER_REGISTRY + script: + - docker pull $DOCKER_ORGANIZATION/barometer-collectd-tests:latest || true + - docker pull $DOCKER_ORGANIZATION/barometer-collectd-tests-base:latest || true + - > + docker build + -t $DOCKER_ORGANIZATION/barometer-collectd-tests-base + -f docker/barometer-collectd-plugin-tests/Dockerfile.base + --network=host + . + - > + docker build + -t $DOCKER_ORGANIZATION/barometer-collectd-tests + -f docker/barometer-collectd-plugin-tests/Dockerfile + --network=host + . + - > + docker run + -t + -v `pwd`/src/collectd/collectd_sample_configs-master:/opt/collectd/etc/collectd.conf.d + -v /var/run:/var/run + -v /tmp:/tmp + -v `pwd`/plugin_test:/tests + --net=host + --privileged + $DOCKER_ORGANIZATION/barometer-collectd-tests:latest + - docker container rm $(sudo docker container ls -aq) + - docker rmi $DOCKER_ORGANIZATION/barometer-collectd-tests:latest + - docker rmi $DOCKER_ORGANIZATION/barometer-collectd-tests-base:latest + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $BAROMETER_SCHEDULE == "true" diff --git a/ci/barometer-build.sh b/ci/barometer-build.sh new file mode 100644 index 00000000..04bbd263 --- /dev/null +++ b/ci/barometer-build.sh @@ -0,0 +1,26 @@ +# This script is used by the barometer-daily CI job in gitlab. +# It builds and packages collectd as an RPM +# After this script is run, the baroemter-daily job runs the +# barometer-upload-artifact.sh script. +set -x + +OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +OPNFV_ARTIFACT_URL="$GS_URL/$OPNFV_ARTIFACT_VERSION/" + +# log info to console +echo "Starting the build of Barometer RPMs" +echo "------------------------------------" +echo + +cd ci +./install_dependencies.sh +./build_rpm.sh +cp utility/rpms_list $WORKSPACE +cd $WORKSPACE + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_ARTIFACT_URL=$OPNFV_ARTIFACT_URL" +) > $WORKSPACE/opnfv.properties + diff --git a/ci/barometer-upload-artifact.sh b/ci/barometer-upload-artifact.sh new file mode 100644 index 00000000..f05dc2af --- /dev/null +++ b/ci/barometer-upload-artifact.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -o nounset +set -o pipefail + +RPM_LIST=$WORKSPACE/rpms_list +RPM_WORKDIR=$WORKSPACE/rpmbuild +RPM_DIR=$RPM_WORKDIR/RPMS/x86_64/ +cd $WORKSPACE/ + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +# Check if all the appropriate RPMs were generated +echo "Checking if all the Barometer RPMs were created" +echo "-----------------------------------------------" +echo + +if [ -d $RPM_DIR ] +then + ls $RPM_DIR > list_of_gen_pack +else + echo "Can't access folder $RPM_DIR with rpm packages" + echo "Barometer nightly build FAILED" + exit 1 +fi + +for PACKAGENAME in `cat $RPM_LIST` +do + if ! grep -q $PACKAGENAME list_of_gen_pack + then + echo "$PACKAGENAME is missing" + echo "Barometer nightly build FAILED" + exit 2 + fi +done + +#remove the file you no longer need. +rm list_of_gen_pack + +echo "Uploading the barometer RPMs to artifacts.opnfv.org" +echo "---------------------------------------------------" +echo + +gsutil -m cp -r $RPM_DIR/* gs://$OPNFV_ARTIFACT_URL > $WORKSPACE/gsutil.log 2>&1 + +# Check if the RPMs were pushed +gsutil ls gs://$OPNFV_ARTIFACT_URL > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading barometer RPMs to gs://$OPNFV_ARTIFACT_URL!" + echo "Check log $WORKSPACE/gsutil.log on the appropriate build server" + exit 1 +else + # upload property files only if build is successful + gsutil cp $WORKSPACE/opnfv.properties gs://$OPNFV_ARTIFACT_URL/opnfv.properties > gsutil.properties.log 2>&1 + gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +fi + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$OPNFV_ARTIFACT_URL/*.rpm > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$OPNFV_ARTIFACT_URL/opnfv.properties > /dev/null 2>&1 + +echo +echo "--------------------------------------------------------" +echo "Done!" +echo "Artifact is available at $OPNFV_ARTIFACT_URL" + +#cleanup the RPM repo from the build machine. +rm -rf $RPM_WORKDIR diff --git a/docker/ansible/default.inv b/docker/ansible/default.inv index e254c26a..d65e1c0e 100644 --- a/docker/ansible/default.inv +++ b/docker/ansible/default.inv @@ -13,8 +13,8 @@ localhost [collectd_hosts:vars] install_mcelog=true insert_ipmi_modules=true -#to use master or experimental container set the collectd flavor below -#possible values: stable|master|experimental +#to use latest (collectd-main branch) or experimental (main + PRs) container +#set the collectd flavor below. Possible values: stable|latest|experimental flavor=stable [influxdb_hosts] diff --git a/docker/ansible/roles/config_files/tasks/snmp_agent.yml b/docker/ansible/roles/config_files/tasks/snmp_agent.yml index c72aee48..5e1ba975 100644 --- a/docker/ansible/roles/config_files/tasks/snmp_agent.yml +++ b/docker/ansible/roles/config_files/tasks/snmp_agent.yml @@ -21,9 +21,9 @@ tags: - snmp -- name: enable snmp agent for master container +- name: enable snmp agent for non-stable container template: - src: master/snmp_agent.conf.j2 + src: latest/snmp_agent.conf.j2 dest: "{{ config_file_dir }}/snmp_agent.conf" when: flavor|default('stable')|string != 'stable' tags: diff --git a/docker/ansible/roles/config_files/templates/master/snmp_agent.conf.j2 b/docker/ansible/roles/config_files/templates/latest/snmp_agent.conf.j2 index 1ff8b228..1ff8b228 100644 --- a/docker/ansible/roles/config_files/templates/master/snmp_agent.conf.j2 +++ b/docker/ansible/roles/config_files/templates/latest/snmp_agent.conf.j2 diff --git a/docs/release/release-notes/index.rst b/docs/release/release-notes/index.rst index 9eb1348e..df4696a5 100644 --- a/docs/release/release-notes/index.rst +++ b/docs/release/release-notes/index.rst @@ -11,5 +11,6 @@ Anuket Barometer Release Notes .. toctree:: :maxdepth: 1 + kali-release-notes release-notes diff --git a/docs/release/release-notes/kali-release-notes.rst b/docs/release/release-notes/kali-release-notes.rst new file mode 100644 index 00000000..250becc1 --- /dev/null +++ b/docs/release/release-notes/kali-release-notes.rst @@ -0,0 +1,40 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +============================ +Barometer Kali Release Notes +============================ + +This document provides the release notes for Kali release of Barometer. + +Summary +------- +The Kali release is the first one since becoming part of Anuket, and focussed +on changes that will make testing and integrating easier. + +Details +------- +Testing and build tools were developed and updated to do the following: + +* A new reference container was added for the collectd-6.0 version, which is + under development and represents a big API change that is not backwards + compatible. This reference build should facilitate porting the plugins that + were previously developed by the Barometer project. + https://jira.anuket.io/browse/BAROMETER-184 + +* Updated to the stable version of collectd to collectd 5.12. + +* Removed duplication in the three existing containers (stable, latest and experimental). + https://jira.anuket.io/browse/BAROMETER-179 + +Some work was started but not completed in the Kali release: + +* Updating of the ansible playbooks for generating configs so that they will be + easier to maintain and extend in the future. + +* Additional testing tools for verifying plugin functionality + +References +---------- +* `Barometer Kali release plan <https://wiki.anuket.io/display/HOME/Barometer+Kali+Release+Planning>`_ +* `Kali Release on Jira <https://jira.anuket.io/projects/BAROMETER/versions/10224>`_ diff --git a/systems/centos/8/build_base_machine.sh b/systems/centos/8/build_base_machine.sh index ceb583d7..1a590841 100755 --- a/systems/centos/8/build_base_machine.sh +++ b/systems/centos/8/build_base_machine.sh @@ -28,17 +28,14 @@ dnf -y update # For collectd dnf install -y yum-utils -dnf install -y epel-release dnf install -y centos-release-opstools # For CentOS 8, a lot of the dependencies are from PowerTools repo dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools -dnf builddep -y collectd - -# CentOS 8 doesn't have intel-cmt-cat-devel packaged, so use the version from CentOS7 -dnf install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/intel-cmt-cat-3.0.1-1.el7.x86_64.rpm \ - http://mirror.centos.org/centos/7/os/x86_64/Packages/intel-cmt-cat-devel-3.0.1-1.el7.x86_64.rpm +# Use collectd.spec from centos-opstools to install deps since +# ``dnf builddep -y collectd`` isn't finding collectd in centos-opstools +dnf builddep -y https://raw.githubusercontent.com/centos-opstools/collectd/master/collectd.spec # Install required packages dnf -y install $(echo " @@ -79,16 +76,15 @@ librdkafka-devel yajl-devel protobuf-c-devel rrdtool-devel -dpdk-19.11.3 +dpdk-20.11 qpid-proton-c-devel # ping collectd-6 liboping-devel -#install epel release required for git-review -epel-release -python3-libvirt python3-pip python36-devel numactl-devel +intel-cmt-cat +intel-cmt-cat-devel " | grep -v ^#) |