Pass VM IDs to monitor for correlation

JIRA: VES-1
Update collectd conf for current agent
Remove unneeded/fake fields in HTTP request rate report

Change-Id: I1e30d5c78ea5025c864a8ca2955adc8a8852ed50
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

4 files changed, 127 insertions, 41 deletions

diff --git a/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c b/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
index ecd4961..49ad3dc 100644
--- a/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
+++ b/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
@@ -252,24 +252,8 @@ void measure_traffic() {
     if (measurement != NULL) {
       cpu();
       evel_measurement_type_set(measurement, "HTTP request rate");
-      evel_measurement_agg_cpu_use_set(measurement, loadavg);
-      evel_measurement_cpu_use_add(measurement, "cpu0", loadavg);
-      evel_measurement_fsys_use_add(measurement,"00-11-22",100.11, 100.22, 33,
-                                                           200.11, 200.22, 44);
-      evel_measurement_fsys_use_add(measurement,"33-44-55",300.11, 300.22, 55,
-                                                           400.11, 400.22, 66);
-
-      evel_measurement_latency_add(measurement, 0.0, 10.0, 20);
-      evel_measurement_latency_add(measurement, 10.0, 20.0, 30);
-
-      evel_measurement_vnic_use_add(measurement, "eth0", 1, 2,
-                                                         3, 4,
-                                                         5, 6,
-                                                         7, 8);
-      evel_measurement_vnic_use_add(measurement, "eth1", 11, 12,
-                                                         13, 14,
-                                                         15, 16,
-                                                         17, 18);
+//      evel_measurement_agg_cpu_use_set(measurement, loadavg);
+//      evel_measurement_cpu_use_add(measurement, "cpu0", loadavg);
 
       evel_rc = evel_post_event((EVENT_HEADER *)measurement);
       if (evel_rc != EVEL_SUCCESS) {
diff --git a/tests/blueprints/tosca-vnfd-hello-ves/monitor.py b/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
index a9e1b65..b34bed9 100644
--- a/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
+++ b/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
@@ -34,11 +34,11 @@ import json
 import jsonschema
 import select
 
-report_time = ''
-requestRate = ''
 monitor_mode = "f"
-summary = ['***** Summary of key stats *****','','','','']
-status = ['','unknown','unknown','unknown','unknown']
+vdu_id = ['','','','']
+summary_e = ['***** Summary of key stats *****','','','']
+summary_c = ['Collectd agents:']
+status = ['','Started','Started','Started']
 base_url = ''
 template_404 = b'''POST {0}'''
 columns = 0
@@ -90,10 +90,14 @@ class PathDispatcher:
 #--------------------------------------------------------------------------
 def process_event(e):
   global status
-  global summary
+  global summary_e
+  global summary_c
+  global vdu_id
   vdu = 0
 
+
   epoch = e.event.commonEventHeader.lastEpochMicrosec
+  sourceId = e.event.commonEventHeader.sourceId
 
   report_time = time.strftime('%Y-%m-%d %H:%M:%S', 
                   time.localtime(int(epoch)/1000000))
@@ -107,17 +111,32 @@ def process_event(e):
 
   if domain == 'measurementsForVfScaling':
     if vdu >= 1:
-      aggregateCpuUsage = e.event.measurementsForVfScaling.aggregateCpuUsage
       requestRate = e.event.measurementsForVfScaling.requestRate
-      summary[vdu] = host + ": state=" + status[vdu] + ", tps=" + str(requestRate) + ", cpu=" + str(aggregateCpuUsage)
+      summary_e[vdu] = host + ": state=" + status[vdu] + ", tps=" + str(requestRate)
     else:
       aggregateCpuUsage = e.event.measurementsForVfScalingFields.aggregateCpuUsage
-      summary[4] = host + ": cpu=" + str(aggregateCpuUsage)
-
-  for s in summary:
+      vNicUsageArray = e.event.measurementsForVfScalingFields.vNicUsageArray
+      s = ""
+      for i in range(1,len(vdu_id)):
+        if sourceId.upper() in vdu_id[i].upper():
+          s = "(VDU"+ str(i) + ") "
+      if s:
+        s += host + ": cpu=" + str(aggregateCpuUsage)
+        found = False
+        for i in range(1,len(summary_c)):
+          if host in summary_c[i]:
+            summary_c[i] = s
+            found = True
+            break
+        if not found:
+          summary_c.extend([s])
+
+  for s in summary_e:
+    print '{0}'.format(s)
+  for s in summary_c:
     print '{0}'.format(s)
 
-  if domain == 'fault':
+  if domain == 'fault' and vdu >= 1:
     alarmCondition = e.event.faultFields.alarmCondition
     specificProblem = e.event.faultFields.specificProblem
 #    status[vdu] = e.event.faultFields.vfStatus
@@ -187,6 +206,7 @@ def ves_monitor(environ, start_response):
 def main(argv=None):
   global columns
   global rows
+
   a,b = os.popen('stty size', 'r').read().split()
   rows = int(a)
   columns = int(b)
@@ -252,6 +272,7 @@ def main(argv=None):
                                 vars=overrides)
     global vel_username
     global vel_password
+    global vdu_id
     vel_username = config.get(config_section,
                               'vel_username',
                               vars=overrides)
@@ -264,6 +285,18 @@ def main(argv=None):
     base_schema_file = config.get(config_section,
                              'base_schema_file',
                               vars=overrides)
+    vdu_id[1] = config.get(config_section,
+                           'vdu1_id',
+                            vars=overrides)
+    vdu_id[2] = config.get(config_section,
+                           'vdu2_id',
+                            vars=overrides)
+    vdu_id[3] = config.get(config_section,
+                           'vdu3_id',
+                            vars=overrides)
+    base_schema_file = config.get(config_section,
+                             'base_schema_file',
+                              vars=overrides)
 
     #----------------------------------------------------------------------
     # Perform some basic error checking on the config.
diff --git a/tests/blueprints/tosca-vnfd-hello-ves/start.sh b/tests/blueprints/tosca-vnfd-hello-ves/start.sh
index 4f53a6a..db5a92f 100755
--- a/tests/blueprints/tosca-vnfd-hello-ves/start.sh
+++ b/tests/blueprints/tosca-vnfd-hello-ves/start.sh
@@ -24,8 +24,7 @@
 #   type: type of VNF component [webserver|lb|monitor|collectd]
 #     webserver params: ID CollectorIP username password
 #     lb params:        ID CollectorIP username password app1_ip app2_ip
-#     collector params: ID CollectorIP username password
-#     collector params: ID CollectorIP username password
+#     monitor params: VDU1_ID VDU1_ID VDU1_ID username password
 #   ID: VM ID
 #   CollectorIP: IP address of the collector
 #   username: Username for Collector RESTful API authentication
@@ -33,6 +32,7 @@
 #   app1_ip app2_ip: address of the web servers
 
 setup_collectd () {
+  guest=$1
   echo "$0: Install prerequisites"
   sudo apt-get update
   echo "$0: Install collectd plugin"
@@ -45,7 +45,9 @@ setup_collectd () {
   sudo sed -i -- "s/#LoadPlugin disk/LoadPlugin disk/" /etc/collectd/collectd.conf
   sudo sed -i -- "s/#LoadPlugin interface/LoadPlugin interface/" /etc/collectd/collectd.conf
   sudo sed -i -- "s/#LoadPlugin memory/LoadPlugin memory/" /etc/collectd/collectd.conf
-  cat <<EOF | sudo tee -a  /etc/collectd/collectd.conf
+
+  if [[ "$guest" == true ]]: then
+    cat <<EOF | sudo tee -a  /etc/collectd/collectd.conf
 <LoadPlugin python>
   Globals true
 </LoadPlugin>
@@ -63,6 +65,46 @@ setup_collectd () {
   Username "hello"
   Password "world"
   FunctionalRole "Collectd VES Agent"
+  GuestRunning true
+</Module>
+</Plugin>
+<Plugin cpu>
+        ReportByCpu false
+        ValuesPercentage true
+</Plugin>
+LoadPlugin aggregation
+<Plugin aggregation>
+        <Aggregation>
+                Plugin "cpu"
+                Type "percent"
+                GroupBy "Host"
+                GroupBy "TypeInstance"
+                SetPlugin "cpu-aggregation"
+                CalculateAverage true
+        </Aggregation>
+</Plugin>
+LoadPlugin uuid
+EOF
+  else 
+    cat <<EOF | sudo tee -a  /etc/collectd/collectd.conf
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+<Plugin python>
+  ModulePath "/home/ubuntu/OpenStackBarcelonaDemo/ves_plugin/"
+  LogTraces true
+  Interactive false
+  Import "ves_plugin"
+<Module ves_plugin>
+  Domain "$collector_ip"
+  Port 30000
+  Path ""
+  Topic ""
+  UseHttps false
+  Username "hello"
+  Password "world"
+  FunctionalRole "Collectd VES Agent"
+  GuestRunning $guest
 </Module>
 </Plugin>
 LoadPlugin virt
@@ -75,7 +117,18 @@ LoadPlugin virt
         ReportByCpu false
         ValuesPercentage true
 </Plugin>
-EOF
+LoadPlugin aggregation
+<Plugin aggregation>
+        <Aggregation>
+                Plugin "cpu"
+                Type "percent"
+                GroupBy "Host"
+                GroupBy "TypeInstance"
+                SetPlugin "cpu-aggregation"
+                CalculateAverage true
+        </Aggregation>
+</Plugin>EOF
+  fi
   sudo service collectd restart
 }
 
@@ -97,8 +150,9 @@ setup_agent () {
   cp ves/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c evel-library/code/evel_demo/evel_demo.c
   
   echo "$0: Update parameters and build agent demo"
+  # This sed command will add a line after the search line 
   sed -i -- "/api_secure,/{n;s/.*/                      \"$username\",/}" evel-library/code/evel_demo/evel_demo.c
-  sed -i -- "/\"hello\",/{n;s/.*/                      \"$password\",/}" evel-library/code/evel_demo/evel_demo.c
+  sed -i -- "/\"$username\",/{n;s/.*/                      \"$password\",/}" evel-library/code/evel_demo/evel_demo.c
 
   echo "$0: Build evel_demo agent"
   cd evel-library/bldjobs
@@ -107,6 +161,9 @@ setup_agent () {
   
   echo "$0: Start evel_demo agent"
   nohup ../output/x86_64/evel_demo --id $vm_id --fqdn $collector_ip --port 30000 --username $username --password $password > /dev/null 2>&1 &
+
+  echo "$0: Start collectd agent running in the VM"
+  setup_collectd true
 }
 
 setup_webserver () {
@@ -200,17 +257,29 @@ setup_monitor () {
   git clone https://github.com/att/evel-test-collector.git
   sed -i -- "s/vel_username = /vel_username = $username/" evel-test-collector/config/collector.conf
   sed -i -- "s/vel_password = /vel_password = $password/" evel-test-collector/config/collector.conf
+  sed -i -- "/vel_topic_name = /a vdu3_id = $vdu3_id" evel-test-collector/config/collector.conf
+  sed -i -- "/vel_topic_name = /a vdu2_id = $vdu2_id" evel-test-collector/config/collector.conf
+  sed -i -- "/vel_topic_name = /a vdu1_id = $vdu1_id" evel-test-collector/config/collector.conf
 
   python monitor.py --config evel-test-collector/config/collector.conf --section default 
 }
 
 type=$1
-vm_id=$2
-collector_ip=$3
-username=$4
-password=$5
-app1_ip=$6
-app2_ip=$7
+
+if [[ "$type" == "monitor" ]]; then
+  vdu1_id=$2
+  vdu2_id=$3
+  vdu3_id=$4
+  username=$5
+  password=$6
+else
+  vm_id=$2
+  collector_ip=$3
+  username=$4
+  password=$5
+  app1_ip=$6
+  app2_ip=$7
+fi
 
 setup_$type
 exit 0
diff --git a/tests/vHello_VES.sh b/tests/vHello_VES.sh
index 9ac4aa2..e8911e7 100644
--- a/tests/vHello_VES.sh
+++ b/tests/vHello_VES.sh
@@ -252,7 +252,7 @@ EOF
   echo "$0: $(date) start Monitor in VDU4 at ${vdu_ip[4]}"
   # Replacing the default collector with monitor.py which has processing logic as well
   scp -i /tmp/tacker/vHello.pem -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/tacker/blueprints/tosca-vnfd-hello-ves/monitor.py ubuntu@${vdu_ip[4]}:/home/ubuntu/monitor.py
-  ssh -i /tmp/tacker/vHello.pem -t -t -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[4]} "bash /home/ubuntu/start.sh monitor ${vdu_id[4]} ${vdu_ip[4]} hello world"
+  ssh -i /tmp/tacker/vHello.pem -t -t -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[4]} "bash /home/ubuntu/start.sh monitor ${vdu_id[1]} ${vdu_id[2]} ${vdu_id[3]} hello world"
 
 #  echo "$0: $(date) verify vHello server is running at http://${vdu_ip[3]}"
 #  apt-get install -y curl