VES YAML: Add node VES message definition

Added new YAML configuration, which allows the VES app
to generate the VES messages for guest,host and hypervisor
separately.

- Fix bug in `INDEX-KEY` definition
- Verbose error messages

Change-Id: I1dddcfc79b27dc68249465f287be34519096c546
Signed-off-by: Mytnyk, Volodymyr <volodymyrx.mytnyk@intel.com>

4 files changed, 481 insertions, 9 deletions

diff --git a/3rd_party/collectd-ves-app/ves_app/guest.yaml b/3rd_party/collectd-ves-app/ves_app/guest.yaml
index c26bd08c..0f48b5e2 100644
--- a/3rd_party/collectd-ves-app/ves_app/guest.yaml
+++ b/3rd_party/collectd-ves-app/ves_app/guest.yaml
@@ -223,6 +223,7 @@ Host Measurements: !Measurements
                       plugin_instance: "{vl.plugin_instance}"
                       type: disk_io_time
                       ds_name: io_time
+                  - DEFAULT: 0
                 diskMergedReadLast: !ValueItem
                   - SELECT:
                       plugin: disk
@@ -243,12 +244,14 @@ Host Measurements: !Measurements
                       plugin_instance: "{vl.plugin_instance}"
                       type: disk_time
                       ds_name: read
+                  - DEFAULT: 0
                 diskTimeWriteLast: !ValueItem
                   - SELECT:
                       plugin: disk
                       plugin_instance: "{vl.plugin_instance}"
                       type: disk_time
                       ds_name: write
+                  - DEFAULT: 0
   - SELECT:
       plugin: memory
       type_instance: free
diff --git a/3rd_party/collectd-ves-app/ves_app/host.yaml b/3rd_party/collectd-ves-app/ves_app/hypervisor.yaml
index 0aa53f4c..0aa53f4c 100644
--- a/3rd_party/collectd-ves-app/ves_app/host.yaml
+++ b/3rd_party/collectd-ves-app/ves_app/hypervisor.yaml
diff --git a/3rd_party/collectd-ves-app/ves_app/node.yaml b/3rd_party/collectd-ves-app/ves_app/node.yaml
new file mode 100644
index 00000000..24f2c638
--- /dev/null
+++ b/3rd_party/collectd-ves-app/ves_app/node.yaml
@@ -0,0 +1,468 @@
+---
+# Common event header definition (required fields and defaults)
+commonEventHeader: &commonEventHeader
+  domain: N/A
+  eventId: "{system.id}"
+  eventName: ""
+  eventType: Info
+  lastEpochMicrosec: 0
+  priority: Normal
+  reportingEntityId: &reportingEntityId "{system.hostname}"
+  reportingEntityName: *reportingEntityId
+  sequence: 0
+  sourceName: N/A
+  startEpochMicrosec: 0
+  version: 2.0
+
+# Host measurements definition
+Host Measurements: !Measurements
+  - ITEM-DESC:
+      event:
+        commonEventHeader: &hostCommonEventHeader
+          <<: *commonEventHeader
+          eventType: hostOS
+          domain: measurementsForVfScaling
+          sourceId: &sourceId "{vl.host}"
+          sourceName: *sourceId
+          startEpochMicrosec: !Number "{vl.time}"
+        measurementsForVfScalingFields: &hostMeasurementsForVfScalingFields
+          measurementsForVfScalingVersion: 2.0
+          additionalMeasurements: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: "/^(?!memory|cpu|interface|disk).*$/"
+            - INDEX-KEY:
+                - host
+                - plugin
+                - plugin_instance
+            - ITEM-DESC:
+                name: !StripExtraDash "{vl.plugin}-{vl.plugin_instance}"
+                arrayOfFields: !ArrayItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: "{vl.plugin}"
+                      plugin_instance: "{vl.plugin_instance}"
+                  - ITEM-DESC:
+                      name: !StripExtraDash "{vl.type}-{vl.type_instance}-{vl.ds_name}"
+                      value: "{vl.value}"
+          measurementInterval: !Number "{vl.interval}"
+          memoryUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: memory
+                type: memory
+                type_instance: free
+            - ITEM-DESC:
+                vmIdentifier: "{vl.host}"
+                memoryFree: !Number "{vl.value}"
+                memoryUsed: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: used
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                memoryBuffered: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: buffered
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                memoryCached: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: cached
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                memorySlabRecl: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: slab_recl
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                  - DEFAULT: 0
+                memorySlabUnrecl: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: slab_unrecl
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                  - DEFAULT: 0
+          cpuUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: cpu
+                type: percent
+                type_instance: idle
+            - ITEM-DESC:
+                cpuIdentifier: "{vl.plugin_instance}"
+                cpuIdle: !Number "{vl.value}"
+                percentUsage: 0.0
+                cpuUsageUser: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: user
+                cpuWait: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: wait
+                cpuUsageInterrupt: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: interrupt
+                cpuUsageNice: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: nice
+                cpuUsageSoftIrq: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: softirq
+                cpuUsageSteal: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: steal
+                cpuUsageSystem: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: system
+          vNicPerformanceArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: interface
+                type: if_packets
+                ds_name: rx
+            - ITEM-DESC:
+                valuesAreSuspect: "true"
+                vNicIdentifier: "{vl.plugin_instance}"
+                receivedTotalPacketsAccumulated: !Number "{vl.value}"
+                transmittedTotalPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_packets
+                      ds_name: tx
+                receivedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      ds_name: rx
+                transmittedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      ds_name: tx
+                receivedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      ds_name: rx
+                transmittedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      ds_name: tx
+                receivedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      ds_name: rx
+                transmittedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      ds_name: tx
+          diskUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: disk
+                type: disk_octets
+                ds_name: read
+            - ITEM-DESC:
+                diskIdentifier: "{vl.plugin_instance}"
+                diskOctetsReadLast: !Number "{vl.value}"
+                diskOctetsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_octets
+                      ds_name: write
+                diskOpsReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      ds_name: read
+                diskOpsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      ds_name: write
+                diskIoTimeLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_io_time
+                      ds_name: io_time
+                  - DEFAULT: 0
+                diskMergedReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_merged
+                      ds_name: read
+                  - DEFAULT: 0
+                diskMergedWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_merged
+                      ds_name: write
+                  - DEFAULT: 0
+                diskTimeReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_time
+                      ds_name: read
+                  - DEFAULT: 0
+                diskTimeWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_time
+                      ds_name: write
+                  - DEFAULT: 0
+  - SELECT:
+      plugin: memory
+      plugin_instance: host
+      type_instance: free
+
+# Guest measurements definition
+Guest Measurements: !Measurements
+  - ITEM-DESC:
+      event:
+        commonEventHeader:
+          <<: *hostCommonEventHeader
+          eventType: guestOS
+        measurementsForVfScalingFields: *hostMeasurementsForVfScalingFields
+  - SELECT:
+      plugin: memory
+      plugin_instance: guest
+      type_instance: free
+
+# Hypervisor measurements definition
+Hypervisor Measurements: !Measurements
+  - ITEM-DESC:
+      event:
+        commonEventHeader:
+          <<: *commonEventHeader
+          eventType: hypervisor
+          domain: measurementsForVfScaling
+          sourceId: &hypervisor_sourceId "{vl.plugin_instance}"
+          sourceName: *hypervisor_sourceId
+          startEpochMicrosec: !Number "{vl.time}"
+        measurementsForVfScalingFields:
+          measurementsForVfScalingVersion: 2.0
+          additionalMeasurements: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: "/^(?!memory|virt_vcpu|disk_octets|disk_ops|if_packets|if_errors|if_octets|if_dropped).*$/"
+            - INDEX-KEY:
+                - host
+                - plugin_instance
+                - type
+            - ITEM-DESC:
+                name: "{vl.type}"
+                arrayOfFields: !ArrayItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: "{vl.type}"
+                  - ITEM-DESC:
+                      name: !StripExtraDash "{vl.type}-{vl.type_instance}-{vl.ds_name}"
+                      value: "{vl.value}"
+          measurementInterval: !Number "{vl.interval}"
+          memoryUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: memory
+                type_instance: total
+            - ITEM-DESC:
+                memoryConfigured: !Bytes2Kibibytes "{vl.value}"
+                vmIdentifier: "{vl.plugin_instance}"
+                memoryUsed: 0.0
+                memoryFree: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: memory
+                      type_instance: rss
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                  - DEFAULT: 0
+          cpuUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: virt_vcpu
+            - ITEM-DESC:
+                cpuIdentifier: "{vl.type_instance}"
+                percentUsage: !Number "{vl.value}"
+          vNicPerformanceArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: if_packets
+                ds_name: rx
+            - ITEM-DESC:
+                valuesAreSuspect: "true"
+                vNicIdentifier: "{vl.type_instance}"
+                receivedTotalPacketsAccumulated: !Number "{vl.value}"
+                transmittedTotalPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_packets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+                receivedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: rx
+                transmittedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+                receivedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      type_instance: "{vl.type_instance}"
+                      ds_name: rx
+                transmittedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+                receivedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      type_instance: "{vl.type_instance}"
+                      ds_name: rx
+                transmittedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+          diskUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: disk_octets
+                ds_name: read
+            - ITEM-DESC:
+                diskIdentifier: "{vl.type_instance}"
+                diskOctetsReadLast: !Number "{vl.value}"
+                diskOctetsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_octets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: write
+                diskOpsReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      type_instance: "{vl.type_instance}"
+                      ds_name: read
+                diskOpsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      type_instance: "{vl.type_instance}"
+                      ds_name: write
+  - SELECT:
+      plugin: virt
+      type_instance: virt_cpu_total
diff --git a/3rd_party/collectd-ves-app/ves_app/normalizer.py b/3rd_party/collectd-ves-app/ves_app/normalizer.py
index 84de5e33..dcb02f27 100644
--- a/3rd_party/collectd-ves-app/ves_app/normalizer.py
+++ b/3rd_party/collectd-ves-app/ves_app/normalizer.py
@@ -241,7 +241,7 @@ class CollectdNotification(CollectdData):
         self.message = message
 
     def __repr__(self):
-        return '{}(host={}, plugin={}, plugin_instance={}, type={},' \
+        return '{}(host={}, plugin={}, plugin_instance={}, type={}, ' \
                'type_instance={}, severity={}, message={}, time={})'.format(
                    self.__class__.__name__, self.host, self.plugin,
                    self.plugin_instance, self.type, self.type_instance,
@@ -274,7 +274,7 @@ class CollectdValue(CollectdData):
                              self.type, self.type_instance, self.ds_name)
 
     def __repr__(self):
-        return '{}(host={}, plugin={}, plugin_instance={}, type={},' \
+        return '{}(host={}, plugin={}, plugin_instance={}, type={}, ' \
                'type_instance={}, ds_name={}, value={}, time={})'.format(
                    self.__class__.__name__, self.host, self.plugin,
                    self.plugin_instance, self.type, self.type_instance,
@@ -347,14 +347,15 @@ class ValueItem(Item):
         # select collectd metric based on SELECT condition
         metrics = loader.collector.items(select)
         assert len(metrics) < 2, \
-            'Wrong SELECT condition, selected {} metrics'.format(len(metrics))
+            'Wrong SELECT condition {}, selected {} metrics'.format(
+            select, len(metrics))
         if len(metrics) > 0:
             item = cls.format_node(value_desc, {'vl': metrics[0],
                                    'system': loader.system})
             return loader.construct_object(item)
         # nothing has been found by SELECT condition, set to DEFAULT value.
-        assert default is not None, \
-            "No metrics selected by SELECT condition and DEFAULT key isn't set"
+        assert default is not None, "No metrics selected by SELECT condition" \
+            " {} and DEFAULT key isn't set".format(select)
         return default
 
 
@@ -398,10 +399,10 @@ class ArrayItem(Item):
         if len(index_keys) > 0:
             metric_set = set()
             for metric in metrics:
-                value_params = {}
+                value = CollectdValue()
                 for key in index_keys:
-                    value_params[key] = getattr(metric, key)
-                metric_set.add(CollectdValue(**value_params))
+                    setattr(value, key, getattr(metric, key))
+                metric_set.add(value)
             metrics = list(metric_set)
         # build items based on SELECT and/or INDEX-KEY criteria
         for metric in metrics:
@@ -465,7 +466,7 @@ class StripExtraDash(yaml.YAMLObject):
 
     @classmethod
     def from_yaml(cls, loader, node):
-        return '-'.join([ x for x in node.value.split('-') if len(x) > 0])
+        return '-'.join([x for x in node.value.split('-') if len(x) > 0])
 
 
 class MapValue(yaml.YAMLObject):