Add a playbook that tests collectd 6 vs collectd5

The playbook will:

* build collectd-6, collectd-latest and flask app
  containers
* generate a set of collectd configs
* launch the collectd-6, collectd-latest with the generated configs
* run the flask app which has a http_server that receives metrics from
  collectd v5 and collectd v6
* display the received metrics from both versions of collectd
  Collectd v5 shows PUTVAL
  Collectd v6 shows PUTMETRIC

The playbook takes the following parameters:
* PR (optional)
  The PRID for an upstream collectd pull request that will be
  passed to the collectd 6 container build

* plugin (optional)
  The name of the plugin that is bneing ported
  This will filter the received metrics to show the value passed.

Additional updates to existing roles include:
* run_collectd
  parameterises the collectd container name so that the role can be reused for
  launching multiple collectd containers by passing distinct names.

Change-Id: Ia62196c347387d380aae69e14ed0dd8e2c76d9c9
Signed-off-by: Emma Foley <efoley@redhat.com>

6 files changed, 236 insertions, 4 deletions

diff --git a/docker/ansible/collectd6_test.yml b/docker/ansible/collectd6_test.yml
new file mode 100644
index 00000000..c1a3a8b4
--- /dev/null
+++ b/docker/ansible/collectd6_test.yml
@@ -0,0 +1,143 @@
+---
+# ansible-playbook -e PR=<PRID> -e new_plugin=<plugin> collectd6_test.yml
+
+# As well as passing a PRID, a config command should be passable too since
+# a lot of the plugins have been explicitly disabled in the build.
+- hosts: localhost
+  become: true
+  tasks:
+    - name: Set names for containers to be used for testing
+      set_fact:
+        collectd5_container_name: "bar-collectd-latest"
+        collectd6_container_name: "bar-collectd-6{{ '-' + PR if PR is defined }}"
+        flask_container_name: "test-collectd-5-v-6"
+
+    - name: Remove existing containers
+      docker_container:
+        name: "{{ item }}"
+        state: absent
+        force_kill: yes
+      with_items:
+        - "{{ collectd5_container_name }}"
+        - "{{ collectd6_container_name }}"
+        - "{{ flask_container_name }}"
+
+    - name: Get a list of containers
+      command:
+        docker ps -a
+      register: output
+
+    - name: Confirm that existing test containers were removed
+      assert:
+        that:
+          - 'not "{{ collectd5_container_name }}" in output.stdout'
+          - 'not "{{ collectd6_container_name }}" in output.stdout'
+          - 'not "{{ flask_container_name }}" in output.stdout'
+
+    - name: Build collectd containers
+      include_role:
+        name: build_collectd
+      args:
+        apply:
+          tags:
+            - latest
+            - collectd-6
+            - flask_test
+      vars:
+        COLLECTD_PULL_REQUESTS: "{{ PR | default() }}"
+        COLLECTD_CONFIG_CMD_ARGS: "{{ '--enable-' + new_plugin if new_plugin is defined }}"
+
+    - name: "Set up config for write_http plugin"
+      set_fact:
+        collectd_plugins: "{{ collectd_plugins | default([]) | union(['write_http']) }}"
+        collectd_plugin_write_http_nodes:
+          flask:
+            url: http://localhost:5000
+            format: "Command"
+
+    - name: Generate collectd configs
+      include_role:
+        name: config_files
+
+    # Since I can't skip-tags here, I have to remove the plugins later
+    # TODO(efoley) Add a disable_plugins and enable_plugins list to
+    # roles/config_files, as an alternative to tags.
+    # This alternative is kinda needed anyway, so that it's easier to add extra
+    # plugins instead of using.
+    # ``{{ collectd_plugins | default([]) | union(['the_plugin_i_want_to_enable'])}}``
+    # Tags can stay, since they are convenient, and easier to pass to the
+    # command line than a list of plugins
+    - name: "Remove plugin configs"
+      file:
+        path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf"
+        state: absent
+      with_items:
+        - snmp_agent
+        - intel_pmu
+
+    # TODO(efoley): The path here should be parameterised, to a degree, I don't
+    # want it to be repeated. And I shouldn't assume that this is always going
+    # to be the value (unless it is in vars/main instead of defaults/main)
+    - name: "Remove plugin configs (collectd 6)"
+      file:
+        path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf"
+        state: absent
+      with_items:
+        - csv
+        - network
+        - rrdtool
+        - write_kafka
+        - write_prometheus
+        - logfile
+
+    - debug:
+        var: PR
+
+    - name: Run the collectd-6 container
+      include_role:
+        name: run_collectd
+      vars:
+        collectd_image_name: "opnfv/barometer-collectd-6{{ '-' + PR if PR is defined }}"
+        collectd_container_name: "{{ collectd6_container_name }}"
+
+    - name: Run the collectd-latest container
+      include_role:
+        name: run_collectd
+      vars:
+        collectd_image_name: opnfv/barometer-collectd-latest
+        collectd_container_name: "{{ collectd5_container_name }}"
+
+    - name: Run the flask test container
+      docker_container:
+        name: "{{ flask_container_name }}"
+        image: test-collectd-write_http
+        detach: yes
+        state: started
+          #network_mode: host
+        published_ports: 5000:5000
+
+    - name: Check output for flask app
+      become: true
+      shell: |
+        docker logs {{ flask_container_name }} {{ '| grep "' + new_plugin + '"' if new_plugin is defined  }} | tail -200
+      register: output
+
+    - debug:
+        var: output.stdout_lines
+
+    - name: Get a list of running containers
+      become: true
+      command:
+        docker ps
+      register: output
+
+    - name: Make sure that the expected containers are running
+      assert:
+        that:
+          - '"{{ collectd6_container_name }}" in output.stdout'
+          - '"{{ collectd5_container_name }}" in output.stdout'
+          - '"{{ flask_container_name }}" in output.stdout'
+
+# Create a small report at the end for collectd versions...
+# Update Apply PRs to check out a branch when it is a single PR
+# OR update these playbooks to use the tag way of checking out a PR
diff --git a/docker/ansible/roles/build_collectd/tasks/main.yml b/docker/ansible/roles/build_collectd/tasks/main.yml
index b8d5f4b9..2ab92296 100644
--- a/docker/ansible/roles/build_collectd/tasks/main.yml
+++ b/docker/ansible/roles/build_collectd/tasks/main.yml
@@ -54,7 +54,7 @@
       args:
         COLLECTD_FLAVOR: collectd-6
         COLLECTD_TAG: "{{ COLLECTD_TAG | default('collectd-6.0') }}"
-        COLLECTD_PULL_REQUESTS: "{{ COLLECTD_PULL_REQUESTS | default(omit) }}"
+        COLLECTD_CONFIG_CMD_ARGS: "{{ COLLECTD_CONFIG_CMD_ARGS if COLLECTD_CONFIG_CMD_ARGS is defined }}"
     source: build
   tags:
     - collectd-6
diff --git a/docker/ansible/roles/run_collectd/tasks/main.yml b/docker/ansible/roles/run_collectd/tasks/main.yml
index 2c5d0e67..bf5aabf5 100644
--- a/docker/ansible/roles/run_collectd/tasks/main.yml
+++ b/docker/ansible/roles/run_collectd/tasks/main.yml
@@ -15,7 +15,7 @@
 
 - name: remove bar-collectd container
   docker_container:
-    name: bar-collectd
+    name: "{{ collectd_container_name }}"
     state: absent
   tags:
     - rm_containers
@@ -52,7 +52,7 @@
 
 - name: launch collectd container
   docker_container:
-    name: bar-collectd
+    name: "{{ collectd_container_name }}"
     image: "{{ collectd_image_name }}"
     volumes: "{{ volumes_list }}"
     entrypoint: "/run_collectd.sh"
diff --git a/docker/ansible/roles/run_collectd/vars/main.yml b/docker/ansible/roles/run_collectd/vars/main.yml
index 96c9032e..26007ecf 100644
--- a/docker/ansible/roles/run_collectd/vars/main.yml
+++ b/docker/ansible/roles/run_collectd/vars/main.yml
@@ -13,6 +13,7 @@
 # limitations under the License.
 ---
 
+collectd_container_name: "bar-collectd"
 default_flavor: "{{ flavor|default('stable')|string }}"
 flavor_image_name: "{{
          'barometer-collectd-latest' if (default_flavor == 'master' or default_flavor == 'latest') else
diff --git a/docs/release/release-notes/notes/collectd-5-v-6-testing-cc821b32bad2794c.yaml b/docs/release/release-notes/notes/collectd-5-v-6-testing-cc821b32bad2794c.yaml
new file mode 100644
index 00000000..20013147
--- /dev/null
+++ b/docs/release/release-notes/notes/collectd-5-v-6-testing-cc821b32bad2794c.yaml
@@ -0,0 +1,10 @@
+release_summary: >
+  Testing playbooks were added to compare collectd5 vs collectd6, for the
+  purpose of helping to review new PRs by comparing the generated metrics
+  between versions.
+testing:
+  - |
+    Added a playbook to compare collectd 5 and collectd 6. The playbook uses
+    existing ansible roles to build both collectd 5 and collectd 6 container
+    images, creates a common configuration, then runs the containers and shows
+    the outputs to let the user inspect the metrics and whether they match.
diff --git a/docs/testing/index.rst b/docs/testing/index.rst
index 392b39f4..f763ca64 100644
--- a/docs/testing/index.rst
+++ b/docs/testing/index.rst
@@ -1 +1,79 @@
-.. To be decided
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) Anuket and others
+
+==============================
+Anuket Barometer testing guide
+==============================
+
+This document will describe how to use different tests in this repo.
+
+There are a number of tools and scripts in Barometer that can be used for testing, whether that is during development, building, code reviews, or run regularly in CI.
+Some of the tests are automated, and cover building collectd, others cover particular plugins.
+
+.. TODO: This guide should also include how to manually verify that collectd plugins are working as expected.
+
+.. TODO: There might be some troubleshooting guide in here too.
+
+Porting collectd to version 6
+=============================
+
+Thre is an ansible playbook for building and running collectd 5 and 6 together to compare the collected metrics.
+This is intended to help test porting from collectd 5 to 6, and confirm equivalency across the versions.
+
+The playbook will::
+
+  * build collectd-6, collectd-latest and flask app containers
+  * generate a set of collectd configs
+  * launch the collectd-6, collectd-latest with the generated configs
+  * run the flask app which has a http server that receives metrics from
+    collectd v5 and collectd v6
+  * display the received metrics from both versions of collectd
+    Collectd v5 shows PUTVAL
+    Collectd v6 shows PUTMETRIC
+
+To run this comparison, use the following command::
+
+  $ cd docker/ansible/
+  $ sudo ansible-playbook -i default.inv collectd6_test.yml
+
+The playbook takes the following parameters:
+
+  * PR (optional)
+    The PRID for an upstream collectd pull request that will be
+    passed to the collectd 6 container build
+
+  * plugin (optional)
+    The name of the plugin that is bneing ported
+    This will filter the received metrics to show the value passed.
+
+To run the playbook with these configs, pass the extra var to ansible::
+
+  sudo ansible-playbook -i default.inv -e PR=<PR_ID> -e plugin=<plugin_name> collectd6_test.yml
+
+The metrics can then be viewed by inspecting the container logs or attaching to the container to view the output::
+
+  $ docker attach <webserver-container>
+  $ #OR
+  $ docker logs <webserver-container>
+
+Metrics from collectd 5 will appear preceeded with ``PUTVAL``, and metrics from collectd 6 will appear preceeded by ``PUTMETRIC``.
+
+::
+
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-2048Kb/vmpage_number-free interval=10.000 1629466502.664:0
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-2048Kb/vmpage_number-used interval=10.000 1629466502.664:0
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-1048576Kb/vmpage_number-free interval=10.000 1629466502.664:0
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-mm-1048576Kb/vmpage_number-used interval=10.000 1629466502.664:0
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-2048Kb/vmpage_number-free interval=10.000 1629466502.664:0
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-2048Kb/vmpage_number-used interval=10.000 1629466502.664:0
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-1048576Kb/vmpage_number-used interval=10.000 1629466502.665:0
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/hugepages-node0-1048576Kb/vmpage_number-free interval=10.000 1629466502.665:0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.807 interval=10.000 label:hugepages="mm-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.807 interval=10.000 label:hugepages="mm-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="mm-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="node0-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="node0-2048Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.809 interval=10.000 label:hugepages="node0-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="free" 0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.809 interval=10.000 label:hugepages="node0-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0
+  PUTMETRIC collectd_hugepages_vmpage_number type=GAUGE time=1629466501.808 interval=10.000 label:hugepages="mm-1048576Kb" label:instance="fbae30cc-2f20-11b2-a85c-819293100691" label:type="used" 0