Test Congress Doctor driver support

When running Congress as Inspector implementation, the Monitor has to be
started after starting the Inspector because we need to first ensure the
Doctor datasource is created, otherwise the Monitor cannot get the
Doctor datasource ID at init.

This patch defaults the Inspector to 'sample' and for the time being
functest will run only against 'sample',  not all supported Inspector
types ('sample' and 'congress'). Testing multiple Inspectors in single
functest run would require major additional changes to our test scripts.
It should still be done and addressed in a future patch. This patch
focus on adding testing support against Congress as first step.

One can test against Congress executing for example:
$ INSPECTOR_TYPE=congress INSTALLER_TYPE=local COMPUTE_HOST=compute1 ./run.sh

JIRA: DOCTOR-56

Change-Id: Icebd6fd6ad0c01d511c97e804727ad2a71f742e8
Signed-off-by: Carlos Goncalves <carlos.goncalves@neclab.eu>

2 files changed, 135 insertions, 13 deletions

diff --git a/tests/monitor.py b/tests/monitor.py
index 9e489865..caf4c321 100644
--- a/tests/monitor.py
+++ b/tests/monitor.py
@@ -8,16 +8,23 @@
 ##############################################################################
 
 import argparse
+from datetime import datetime
 import json
+import os
 import requests
 import socket
+import sys
 import time
 
+from congressclient.v1 import client
+from keystoneclient import session as ksc_session
+from keystoneclient.auth.identity import v2
 
 # NOTE: icmp message with all zero data (checksum = 0xf7ff)
 #       see https://tools.ietf.org/html/rfc792
 ICMP_ECHO_MESSAGE = '\x08\x00\xf7\xff\x00\x00\x00\x00'
 
+SUPPORTED_INSPECTOR_TYPES = ['sample', 'congress']
 
 class DoctorMonitorSample(object):
 
@@ -26,10 +33,30 @@ class DoctorMonitorSample(object):
     event_type = "compute.host.down"
 
     def __init__(self, args):
+        if args.inspector_type not in SUPPORTED_INSPECTOR_TYPES:
+            raise Exception("Inspector type '%s' not supported", args.inspector_type)
+
         self.hostname = args.hostname
-        self.inspector = args.inspector
+        self.inspector_url = args.inspector_url
+        self.inspector_type = args.inspector_type
         self.ip_addr = args.ip or socket.gethostbyname(self.hostname)
 
+        if self.inspector_type == 'congress':
+            auth = v2.Password(auth_url=os.environ['OS_AUTH_URL'],
+                               username=os.environ['OS_USERNAME'],
+                               password=os.environ['OS_PASSWORD'],
+                               tenant_name=os.environ['OS_TENANT_NAME'])
+            self.session = ksc_session.Session(auth=auth)
+
+            congress = client.Client(session=self.session, service_type='policy')
+            ds = congress.list_datasources()['results']
+            doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+                             None)
+
+            congress_endpoint = congress.httpclient.get_endpoint(auth=auth)
+            self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' %
+                                  (congress_endpoint, doctor_ds['id']))
+
     def start_loop(self):
         print "start ping to host %(h)s (ip=%(i)s)" % {'h': self.hostname,
                                                        'i': self.ip_addr}
@@ -48,10 +75,33 @@ class DoctorMonitorSample(object):
             time.sleep(self.interval)
 
     def report_error(self):
-        payload = {"type": self.event_type, "hostname": self.hostname}
-        data = json.dumps(payload)
-        headers = {'content-type': 'application/json'}
-        requests.post(self.inspector, data=data, headers=headers)
+        if self.inspector_type == 'sample':
+            payload = {"type": self.event_type, "hostname": self.hostname}
+            data = json.dumps(payload)
+            headers = {'content-type': 'application/json'}
+            requests.post(self.inspector_url, data=data, headers=headers)
+        elif self.inspector_type == 'congress':
+            data = [
+                {
+                    'id': 'monitor_sample_id1',
+                    'time': datetime.now().isoformat(),
+                    'type': self.event_type,
+                    'details': {
+                        'hostname': self.hostname,
+                        'status': 'down',
+                        'monitor': 'monitor_sample',
+                        'monitor_event_id': 'monitor_sample_event1'
+                    },
+                },
+            ]
+
+            headers = {
+                'Content-Type': 'application/json',
+                'Accept': 'application/json',
+                'X-Auth-Token':self.session.get_token(),
+            }
+
+            requests.put(self.inspector_url, data=json.dumps(data), headers=headers)
 
 
 def get_args():
@@ -60,7 +110,10 @@ def get_args():
                         help='a hostname to monitor connectivity')
     parser.add_argument('ip', metavar='IP', type=str, nargs='?',
                         help='an IP address to monitor connectivity')
-    parser.add_argument('inspector', metavar='INSPECTOR', type=str, nargs='?',
+    parser.add_argument('inspector_type', metavar='INSPECTOR_TYPE', type=str, nargs='?',
+                        help='inspector to report',
+                        default='sample')
+    parser.add_argument('inspector_url', metavar='INSPECTOR_URL', type=str, nargs='?',
                         help='inspector url to report error',
                         default='http://127.0.0.1:12345/events')
     return parser.parse_args()
diff --git a/tests/run.sh b/tests/run.sh
index 477d9c2f..965f9786 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -29,6 +29,9 @@ SUPPORTED_INSTALLER_TYPES="apex fuel local"
 INSTALLER_TYPE=${INSTALLER_TYPE:-local}
 INSTALLER_IP=${INSTALLER_IP:-none}
 
+SUPPORTED_INSPECTOR_TYPES="sample congress"
+INSPECTOR_TYPE=${INSPECTOR_TYPE:-sample}
+
 ssh_opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
 as_doctor_user="--os-username $DOCTOR_USER --os-password $DOCTOR_PW
                 --os-tenant-name $DOCTOR_PROJECT"
@@ -38,6 +41,11 @@ if [[ ! "$SUPPORTED_INSTALLER_TYPES" =~ "$INSTALLER_TYPE" ]] ; then
     exit 1
 fi
 
+if [[ ! "$SUPPORTED_INSPECTOR_TYPES" =~ "$INSPECTOR_TYPE" ]] ; then
+    echo "ERROR: INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
+    exit 1
+fi
+
 get_compute_host_info() {
     # get computer host info which VM boot in
     COMPUTE_HOST=$(openstack $as_doctor_user server show $VM_NAME |
@@ -186,7 +194,7 @@ print_log() {
 
 start_monitor() {
     pgrep -f "python monitor.py" && return 0
-    sudo python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" \
+    sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
         "http://127.0.0.1:$INSPECTOR_PORT/events" > monitor.log 2>&1 &
 }
 
@@ -196,15 +204,76 @@ stop_monitor() {
     print_log monitor.log
 }
 
+congress_add_rule() {
+    name=$1
+    policy=$2
+    rule=$3
+
+    if ! openstack congress policy rule list $policy | grep -q -e "// Name: $name$" ; then
+        openstack congress policy rule create --name $name $policy "$rule"
+    fi
+}
+
+congress_del_rule() {
+    name=$1
+    policy=$2
+
+    if openstack congress policy rule list $policy | grep -q -e "^// Name: $name$" ; then
+        openstack congress policy rule delete $policy $name
+    fi
+}
+
+congress_setup_rules() {
+    congress_add_rule host_down classification \
+        'host_down(host) :-
+            doctor:events(hostname=host, type="compute.host.down", status="down")'
+
+    congress_add_rule active_instance_in_host classification \
+        'active_instance_in_host(vmid, host) :-
+            nova:servers(id=vmid, host_name=host, status="ACTIVE")'
+
+    congress_add_rule host_force_down classification \
+        'execute[nova:services.force_down(host, "nova-compute", "True")] :-
+            host_down(host)'
+
+    congress_add_rule error_vm_states classification \
+        'execute[nova:servers.reset_state(vmid, "error")] :-
+            host_down(host),
+            active_instance_in_host(vmid, host)'
+}
+
 start_inspector() {
-    pgrep -f "python inspector.py" && return 0
-    python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
+    if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then
+        pgrep -f "python inspector.py" && return 0
+        python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
+    elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then
+        nova_api_min_version="2.11"
+        nova_api_version=$(openstack congress datasource list | \
+                           grep nova | grep -Po "(?<='api_version': ')[^']*")
+        [[ -z $nova_api_version ]] && nova_api_version="2.0"
+        if [[ "$nova_api_version" < "$nova_api_min_version" ]]; then
+            echo "ERROR: Congress Nova datasource API version < $nova_api_min_version ($nova_api_version)"
+            exit 1
+        fi
+        openstack congress driver list | grep -q " doctor "
+        openstack congress datasource list | grep -q " doctor " || {
+            openstack congress datasource create doctor doctor
+        }
+        congress_setup_rules
+    fi
 }
 
 stop_inspector() {
-    pgrep -f "python inspector.py" || return 0
-    kill $(pgrep -f "python inspector.py")
-    print_log inspector.log
+    if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then
+        pgrep -f "python inspector.py" || return 0
+        kill $(pgrep -f "python inspector.py")
+        print_log inspector.log
+    elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then
+        congress_del_rule host_force_down classification
+        congress_del_rule error_vm_states classification
+        congress_del_rule active_instance_in_host classification
+        congress_del_rule host_down classification
+    fi
 }
 
 start_consumer() {
@@ -364,8 +433,8 @@ get_consumer_ip
 create_alarm
 
 echo "starting doctor sample components..."
-start_monitor
 start_inspector
+start_monitor
 start_consumer
 
 sleep 60