Fix state change detection, add to monitor

JIRA: VES-1

Change-Id: Icd78d6ff941e46850d4635a7d136e8b1a85baa25
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

3 files changed, 68 insertions, 51 deletions

diff --git a/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c b/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
index 39cd528..0eb655c 100644
--- a/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
+++ b/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
@@ -145,31 +145,26 @@ void check_app_container_state() {
   printf("Checking status of app container\n");
   FILE *fp;
   int status;
-  const int JSON_MAX = 1000;
-  char json_line[JSON_MAX];
+  char state[100];
 
-  fp = popen("sudo docker inspect vHello", "r");
+  fp = popen("sudo docker inspect vHello | grep Status | sed -- 's/,//g' | sed -- 's/\"//g' | sed -- 's/            Status: //g'", "r");
   if (fp == NULL) {
     EVEL_ERROR("popen failed to execute command");
   }
 
-  while (fgets(json_line, JSON_MAX, fp) != NULL) {
-    if (strstr(json_line, "Running") != NULL) {
-      if (strstr(json_line, "true") != NULL) {
-        if (strcmp(app_prevstate,"Stopped") == 0) {
-          printf("App state change detected: Started\n");
-          report_app_statechange("Started");
-          app_prevstate = "Running";
-        }
-      }
-      else {
-        if (strcmp(app_prevstate, "Running") == 0) {
-          printf("App state change detected: Stopped\n");
-          report_app_statechange("Stopped");
-          app_prevstate = "Stopped";
-        }
-      }
-      break;
+  fgets(state, 100, fp);
+  if (strstr(state, "running") != NULL) {
+    if (strcmp(app_prevstate,"Stopped") == 0) {
+      printf("App state change detected: Started\n");
+      report_app_statechange("Started");
+      app_prevstate = "Running";
+    }
+  }
+  else {
+    if (strcmp(app_prevstate, "Running") == 0) {
+      printf("App state change detected: Stopped\n");
+      report_app_statechange("Stopped");
+      app_prevstate = "Stopped";
     }
   }
   status = pclose(fp);
@@ -216,7 +211,6 @@ void measure_traffic() {
   if (sec == 0) sec = 59;
   sprintf(secs, "%02d", sec);
   strncat(period, secs, 9);
-  printf("%s\n", period);
 
   strcpy(cmd, "sudo docker logs vHello | grep -c ");
   strncat(cmd, period, 100);
@@ -440,7 +434,7 @@ int main(int argc, char ** argv)
   while (1)
   {
     EVEL_INFO("MAI: Starting main loop");
-    printf("Starting main loop\n");
+//    printf("Starting main loop\n");
 
     printf("Sending heartbeat\n");
     heartbeat = evel_new_heartbeat();
@@ -464,7 +458,7 @@ int main(int argc, char ** argv)
     /* MAIN RETRY LOOP.  Loop every 10 secs.                                 */
     /* TODO: Listener for throttling back scheduled reports.                 */
     /*************************************************************************/
-    printf("End of main loop, sleeping for 10 seconds\n");
+ //   printf("End of main loop, sleeping for 10 seconds\n");
     fflush(stdout);
     sleep(10);
  }
diff --git a/tests/blueprints/tosca-vnfd-hello-ves/monitor.py b/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
index 9f3b7ff..24ad557 100644
--- a/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
+++ b/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
@@ -25,9 +25,13 @@ with open('/home/ubuntu/ves.log') as f:
     if line:
 #      print line,
       if "requestRate" in line:
-#         print line,
-         rate = line[27:-2]
-         print 'request rate: {0}'.format(rate)
+#       print line,
+        rate = line[27:-2]
+        print 'request rate: {0}'.format(rate)
 #....5....1....5....2....5....3....5
 #            "requestRate": 2264,
+      if "\"specificProblem\": \"Started\"" in line:
+        print 'app state change: Started'
+      if "\"specificProblem\": \"Stopped\"" in line:
+        print 'app state change: Stopped'
 
diff --git a/tests/vHello_VES.sh b/tests/vHello_VES.sh
index 95d3f4f..abb9687 100644
--- a/tests/vHello_VES.sh
+++ b/tests/vHello_VES.sh
@@ -21,13 +21,19 @@
 # How to use:
 #   $ git clone https://gerrit.opnfv.org/gerrit/ves
 #   $ cd ves/tests
-#   $ bash vHello_VES.sh [setup|start|run|stop|clean|collector]
+#   $ bash vHello_VES.sh [setup|start|run|test|stop|clean] 
+#        [collector|traffic|pause|nic]
 #   setup: setup test environment
 #   start: install blueprint and run test
 #   run: setup test environment and run test
+#   test: run test tools/scenario - see below
 #   stop: stop test and uninstall blueprint
 #   clean: cleanup after test
-#   collector: attach to the collector VM and run the collector
+#   Test:
+#     collector: attach to the collector VM and run the collector
+#     traffic: generate some traffic
+#     pause: pause the VNF (web server) for a minute to generate a state change
+#     nic: timed ifdown/ifup to generate a NIC fault report
 
 trap 'fail' ERR
 
@@ -240,18 +246,37 @@ EOF
   fail
 }
 
-collector () {
+stop() {
   echo "$0: setup OpenStack CLI environment"
   source /tmp/tacker/admin-openrc.sh
 
-  echo "$0: find Collector VM IP"
-  VDU2_IP=$(openstack server list | awk "/VDU2/ { print \$10 }")
+  echo "$0: uninstall vHello blueprint via CLI"
+  vid=($(tacker vnf-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnf-delete ${id};  done
+  vid=($(tacker vnfd-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnfd-delete ${id};  done
+# Need to remove the floatingip deletion or make it specific to the vHello VM
+#  fip=($(neutron floatingip-list|grep -v "+"|grep -v id|awk '{print $2}')); for id in ${fip[@]}; do neutron floatingip-delete ${id};  done
+  sg=($(openstack security group list|grep vHello|awk '{print $2}'))
+  for id in ${sg[@]}; do try 5 5 "openstack security group delete ${id}";  done
+}
+
+#
+# Test tools and scenarios
+#
+
+get_vdu_ip () {
+  source /tmp/tacker/admin-openrc.sh
+
+  echo "$0: find VM IP for $1"
+  vdu_ip=$(openstack server list | awk "/$1/ { print \$10 }")
+}
 
+collector () {
   echo "$0: Start the VES Collector in VDU2 - Stop first if running"
+  get_vdu_ip VDU2
   sudo cp /tmp/tacker/vHello.pem /tmp/vHello.pem
   sudo chown $USER:$USER /tmp/vHello.pem
   chmod 600 /tmp/vHello.pem
-  ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$VDU2_IP << 'EOF'
+  ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$vdu_ip << 'EOF'
 sudo kill $(ps -ef | grep evel-test-collector | awk '{print $2}')
 cd /home/ubuntu/
 nohup python evel-test-collector/code/collector/collector.py \
@@ -262,31 +287,22 @@ EOF
 }
 
 traffic () {
-  echo "$0: setup OpenStack CLI environment"
-  source /tmp/tacker/admin-openrc.sh
-
-  echo "$0: find Agent VM IP"
-  VDU1_IP=$(openstack server list | awk "/VDU1/ { print \$10 }")
-
   echo "$0: Generate some traffic, somewhat randomly"
+  get_vdu_ip VDU1
   while true
   do
     sleep .0$[ ( $RANDOM % 10 ) + 1 ]s
-    curl -s http://$VDU1_IP > /dev/null
+    curl -s http://$vdu_ip > /dev/null
   done
 }
 
-stop() {
-  echo "$0: setup OpenStack CLI environment"
-  source /tmp/tacker/admin-openrc.sh
-
-  echo "$0: uninstall vHello blueprint via CLI"
-  vid=($(tacker vnf-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnf-delete ${id};  done
-  vid=($(tacker vnfd-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnfd-delete ${id};  done
-# Need to remove the floatingip deletion or make it specific to the vHello VM
-#  fip=($(neutron floatingip-list|grep -v "+"|grep -v id|awk '{print $2}')); for id in ${fip[@]}; do neutron floatingip-delete ${id};  done
-  sg=($(openstack security group list|grep vHello|awk '{print $2}'))
-  for id in ${sg[@]}; do try 5 5 "openstack security group delete ${id}";  done
+pause () {
+  echo "$0: Pause the VNF (web server) for a minute to generate a state change fault report (Stopped)"
+  get_vdu_ip VDU1
+  ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$vdu_ip "sudo docker pause vHello"
+  sleep 60
+  echo "$0: Unpausing the VNF to generate a state change fault report (Started)"
+  ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$vdu_ip "sudo docker unpause vHello"
 }
 
 forward_to_container () {
@@ -315,10 +331,13 @@ case "$1" in
     fi
     pass
     ;;
-  collector|traffic)
+  traffic)
     $1
     pass
     ;;
+  test)
+    $2
+    ;;
   clean)
     echo "$0: Uninstall Tacker and test environment"
     bash /tmp/tacker/tacker-setup.sh $1 clean