diff options
| -rw-r--r-- | tests/monitor.py | 65 | ||||
| -rwxr-xr-x | tests/run.sh | 83 |
2 files changed, 135 insertions, 13 deletions
diff --git a/tests/monitor.py b/tests/monitor.py index 9e489865..caf4c321 100644 --- a/tests/monitor.py +++ b/tests/monitor.py @@ -8,16 +8,23 @@ ############################################################################## import argparse +from datetime import datetime import json +import os import requests import socket +import sys import time +from congressclient.v1 import client +from keystoneclient import session as ksc_session +from keystoneclient.auth.identity import v2 # NOTE: icmp message with all zero data (checksum = 0xf7ff) # see https://tools.ietf.org/html/rfc792 ICMP_ECHO_MESSAGE = '\x08\x00\xf7\xff\x00\x00\x00\x00' +SUPPORTED_INSPECTOR_TYPES = ['sample', 'congress'] class DoctorMonitorSample(object): @@ -26,10 +33,30 @@ class DoctorMonitorSample(object): event_type = "compute.host.down" def __init__(self, args): + if args.inspector_type not in SUPPORTED_INSPECTOR_TYPES: + raise Exception("Inspector type '%s' not supported", args.inspector_type) + self.hostname = args.hostname - self.inspector = args.inspector + self.inspector_url = args.inspector_url + self.inspector_type = args.inspector_type self.ip_addr = args.ip or socket.gethostbyname(self.hostname) + if self.inspector_type == 'congress': + auth = v2.Password(auth_url=os.environ['OS_AUTH_URL'], + username=os.environ['OS_USERNAME'], + password=os.environ['OS_PASSWORD'], + tenant_name=os.environ['OS_TENANT_NAME']) + self.session = ksc_session.Session(auth=auth) + + congress = client.Client(session=self.session, service_type='policy') + ds = congress.list_datasources()['results'] + doctor_ds = next((item for item in ds if item['driver'] == 'doctor'), + None) + + congress_endpoint = congress.httpclient.get_endpoint(auth=auth) + self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' % + (congress_endpoint, doctor_ds['id'])) + def start_loop(self): print "start ping to host %(h)s (ip=%(i)s)" % {'h': self.hostname, 'i': self.ip_addr} @@ -48,10 +75,33 @@ class DoctorMonitorSample(object): time.sleep(self.interval) def report_error(self): - payload = {"type": self.event_type, "hostname": self.hostname} - data = json.dumps(payload) - headers = {'content-type': 'application/json'} - requests.post(self.inspector, data=data, headers=headers) + if self.inspector_type == 'sample': + payload = {"type": self.event_type, "hostname": self.hostname} + data = json.dumps(payload) + headers = {'content-type': 'application/json'} + requests.post(self.inspector_url, data=data, headers=headers) + elif self.inspector_type == 'congress': + data = [ + { + 'id': 'monitor_sample_id1', + 'time': datetime.now().isoformat(), + 'type': self.event_type, + 'details': { + 'hostname': self.hostname, + 'status': 'down', + 'monitor': 'monitor_sample', + 'monitor_event_id': 'monitor_sample_event1' + }, + }, + ] + + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token':self.session.get_token(), + } + + requests.put(self.inspector_url, data=json.dumps(data), headers=headers) def get_args(): @@ -60,7 +110,10 @@ def get_args(): help='a hostname to monitor connectivity') parser.add_argument('ip', metavar='IP', type=str, nargs='?', help='an IP address to monitor connectivity') - parser.add_argument('inspector', metavar='INSPECTOR', type=str, nargs='?', + parser.add_argument('inspector_type', metavar='INSPECTOR_TYPE', type=str, nargs='?', + help='inspector to report', + default='sample') + parser.add_argument('inspector_url', metavar='INSPECTOR_URL', type=str, nargs='?', help='inspector url to report error', default='http://127.0.0.1:12345/events') return parser.parse_args() diff --git a/tests/run.sh b/tests/run.sh index 477d9c2f..965f9786 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -29,6 +29,9 @@ SUPPORTED_INSTALLER_TYPES="apex fuel local" INSTALLER_TYPE=${INSTALLER_TYPE:-local} INSTALLER_IP=${INSTALLER_IP:-none} +SUPPORTED_INSPECTOR_TYPES="sample congress" +INSPECTOR_TYPE=${INSPECTOR_TYPE:-sample} + ssh_opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" as_doctor_user="--os-username $DOCTOR_USER --os-password $DOCTOR_PW --os-tenant-name $DOCTOR_PROJECT" @@ -38,6 +41,11 @@ if [[ ! "$SUPPORTED_INSTALLER_TYPES" =~ "$INSTALLER_TYPE" ]] ; then exit 1 fi +if [[ ! "$SUPPORTED_INSPECTOR_TYPES" =~ "$INSPECTOR_TYPE" ]] ; then + echo "ERROR: INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported." + exit 1 +fi + get_compute_host_info() { # get computer host info which VM boot in COMPUTE_HOST=$(openstack $as_doctor_user server show $VM_NAME | @@ -186,7 +194,7 @@ print_log() { start_monitor() { pgrep -f "python monitor.py" && return 0 - sudo python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" \ + sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \ "http://127.0.0.1:$INSPECTOR_PORT/events" > monitor.log 2>&1 & } @@ -196,15 +204,76 @@ stop_monitor() { print_log monitor.log } +congress_add_rule() { + name=$1 + policy=$2 + rule=$3 + + if ! openstack congress policy rule list $policy | grep -q -e "// Name: $name$" ; then + openstack congress policy rule create --name $name $policy "$rule" + fi +} + +congress_del_rule() { + name=$1 + policy=$2 + + if openstack congress policy rule list $policy | grep -q -e "^// Name: $name$" ; then + openstack congress policy rule delete $policy $name + fi +} + +congress_setup_rules() { + congress_add_rule host_down classification \ + 'host_down(host) :- + doctor:events(hostname=host, type="compute.host.down", status="down")' + + congress_add_rule active_instance_in_host classification \ + 'active_instance_in_host(vmid, host) :- + nova:servers(id=vmid, host_name=host, status="ACTIVE")' + + congress_add_rule host_force_down classification \ + 'execute[nova:services.force_down(host, "nova-compute", "True")] :- + host_down(host)' + + congress_add_rule error_vm_states classification \ + 'execute[nova:servers.reset_state(vmid, "error")] :- + host_down(host), + active_instance_in_host(vmid, host)' +} + start_inspector() { - pgrep -f "python inspector.py" && return 0 - python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 & + if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then + pgrep -f "python inspector.py" && return 0 + python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 & + elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then + nova_api_min_version="2.11" + nova_api_version=$(openstack congress datasource list | \ + grep nova | grep -Po "(?<='api_version': ')[^']*") + [[ -z $nova_api_version ]] && nova_api_version="2.0" + if [[ "$nova_api_version" < "$nova_api_min_version" ]]; then + echo "ERROR: Congress Nova datasource API version < $nova_api_min_version ($nova_api_version)" + exit 1 + fi + openstack congress driver list | grep -q " doctor " + openstack congress datasource list | grep -q " doctor " || { + openstack congress datasource create doctor doctor + } + congress_setup_rules + fi } stop_inspector() { - pgrep -f "python inspector.py" || return 0 - kill $(pgrep -f "python inspector.py") - print_log inspector.log + if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then + pgrep -f "python inspector.py" || return 0 + kill $(pgrep -f "python inspector.py") + print_log inspector.log + elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then + congress_del_rule host_force_down classification + congress_del_rule error_vm_states classification + congress_del_rule active_instance_in_host classification + congress_del_rule host_down classification + fi } start_consumer() { @@ -364,8 +433,8 @@ get_consumer_ip create_alarm echo "starting doctor sample components..." -start_monitor start_inspector +start_monitor start_consumer sleep 60 |