Test Congress Doctor driver support

When running Congress as Inspector implementation, the Monitor has to be
started after starting the Inspector because we need to first ensure the
Doctor datasource is created, otherwise the Monitor cannot get the
Doctor datasource ID at init.

This patch defaults the Inspector to 'sample' and for the time being
functest will run only against 'sample',  not all supported Inspector
types ('sample' and 'congress'). Testing multiple Inspectors in single
functest run would require major additional changes to our test scripts.
It should still be done and addressed in a future patch. This patch
focus on adding testing support against Congress as first step.

One can test against Congress executing for example:
$ INSPECTOR_TYPE=congress INSTALLER_TYPE=local COMPUTE_HOST=compute1 ./run.sh

JIRA: DOCTOR-56

Change-Id: Icebd6fd6ad0c01d511c97e804727ad2a71f742e8
Signed-off-by: Carlos Goncalves <carlos.goncalves@neclab.eu>

1 files changed, 76 insertions, 7 deletions

diff --git a/tests/run.sh b/tests/run.sh
index 477d9c2f..965f9786 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -29,6 +29,9 @@ SUPPORTED_INSTALLER_TYPES="apex fuel local"
 INSTALLER_TYPE=${INSTALLER_TYPE:-local}
 INSTALLER_IP=${INSTALLER_IP:-none}
 
+SUPPORTED_INSPECTOR_TYPES="sample congress"
+INSPECTOR_TYPE=${INSPECTOR_TYPE:-sample}
+
 ssh_opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
 as_doctor_user="--os-username $DOCTOR_USER --os-password $DOCTOR_PW
                 --os-tenant-name $DOCTOR_PROJECT"
@@ -38,6 +41,11 @@ if [[ ! "$SUPPORTED_INSTALLER_TYPES" =~ "$INSTALLER_TYPE" ]] ; then
     exit 1
 fi
 
+if [[ ! "$SUPPORTED_INSPECTOR_TYPES" =~ "$INSPECTOR_TYPE" ]] ; then
+    echo "ERROR: INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
+    exit 1
+fi
+
 get_compute_host_info() {
     # get computer host info which VM boot in
     COMPUTE_HOST=$(openstack $as_doctor_user server show $VM_NAME |
@@ -186,7 +194,7 @@ print_log() {
 
 start_monitor() {
     pgrep -f "python monitor.py" && return 0
-    sudo python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" \
+    sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
         "http://127.0.0.1:$INSPECTOR_PORT/events" > monitor.log 2>&1 &
 }
 
@@ -196,15 +204,76 @@ stop_monitor() {
     print_log monitor.log
 }
 
+congress_add_rule() {
+    name=$1
+    policy=$2
+    rule=$3
+
+    if ! openstack congress policy rule list $policy | grep -q -e "// Name: $name$" ; then
+        openstack congress policy rule create --name $name $policy "$rule"
+    fi
+}
+
+congress_del_rule() {
+    name=$1
+    policy=$2
+
+    if openstack congress policy rule list $policy | grep -q -e "^// Name: $name$" ; then
+        openstack congress policy rule delete $policy $name
+    fi
+}
+
+congress_setup_rules() {
+    congress_add_rule host_down classification \
+        'host_down(host) :-
+            doctor:events(hostname=host, type="compute.host.down", status="down")'
+
+    congress_add_rule active_instance_in_host classification \
+        'active_instance_in_host(vmid, host) :-
+            nova:servers(id=vmid, host_name=host, status="ACTIVE")'
+
+    congress_add_rule host_force_down classification \
+        'execute[nova:services.force_down(host, "nova-compute", "True")] :-
+            host_down(host)'
+
+    congress_add_rule error_vm_states classification \
+        'execute[nova:servers.reset_state(vmid, "error")] :-
+            host_down(host),
+            active_instance_in_host(vmid, host)'
+}
+
 start_inspector() {
-    pgrep -f "python inspector.py" && return 0
-    python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
+    if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then
+        pgrep -f "python inspector.py" && return 0
+        python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
+    elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then
+        nova_api_min_version="2.11"
+        nova_api_version=$(openstack congress datasource list | \
+                           grep nova | grep -Po "(?<='api_version': ')[^']*")
+        [[ -z $nova_api_version ]] && nova_api_version="2.0"
+        if [[ "$nova_api_version" < "$nova_api_min_version" ]]; then
+            echo "ERROR: Congress Nova datasource API version < $nova_api_min_version ($nova_api_version)"
+            exit 1
+        fi
+        openstack congress driver list | grep -q " doctor "
+        openstack congress datasource list | grep -q " doctor " || {
+            openstack congress datasource create doctor doctor
+        }
+        congress_setup_rules
+    fi
 }
 
 stop_inspector() {
-    pgrep -f "python inspector.py" || return 0
-    kill $(pgrep -f "python inspector.py")
-    print_log inspector.log
+    if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then
+        pgrep -f "python inspector.py" || return 0
+        kill $(pgrep -f "python inspector.py")
+        print_log inspector.log
+    elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then
+        congress_del_rule host_force_down classification
+        congress_del_rule error_vm_states classification
+        congress_del_rule active_instance_in_host classification
+        congress_del_rule host_down classification
+    fi
 }
 
 start_consumer() {
@@ -364,8 +433,8 @@ get_consumer_ip
 create_alarm
 
 echo "starting doctor sample components..."
-start_monitor
 start_inspector
+start_monitor
 start_consumer
 
 sleep 60