4 files changed, 351 insertions, 0 deletions
diff --git a/tests/consumer.py b/tests/consumer.py
new file mode 100644
index 00000000..aa6e90e3
--- /dev/null
+++ b/tests/consumer.py
@@ -0,0 +1,48 @@
+#
+# Copyright 2016 NEC Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+from flask import Flask
+from flask import request
+import json
+import os
+import time
+
+
+app = Flask(__name__)
+
+
+@app.route('/failure', methods=['POST'])
+def event_posted():
+    app.logger.debug('doctor consumer notified at %s' % time.time())
+    app.logger.debug('received data = %s' % request.data)
+    d = json.loads(request.data)
+    return "OK"
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Doctor Sample Monitor')
+    parser.add_argument('port', metavar='PORT', type=int, nargs='?',
+                        help='a port for inspectpr')
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    app.run(port=args.port, debug=True)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/inspector.py b/tests/inspector.py
new file mode 100644
index 00000000..2445c77a
--- /dev/null
+++ b/tests/inspector.py
@@ -0,0 +1,76 @@
+#
+# Copyright 2016 NEC Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+from flask import Flask
+from flask import request
+import json
+import os
+
+import novaclient.client as novaclient
+
+
+class DoctorInspectorSample(object):
+
+    nova_api_version = 2.11
+
+    def __init__(self):
+        self.nova = novaclient.Client(self.nova_api_version,
+                                      os.environ['OS_USERNAME'],
+                                      os.environ['OS_PASSWORD'],
+                                      os.environ['OS_TENANT_NAME'],
+                                      os.environ['OS_AUTH_URL'],
+                                      connection_pool=True)
+        # check nova is available
+        self.nova.servers.list(detailed=False)
+
+    def disable_compute_host(self, hostname):
+        opts = {'all_tenants': True, 'host': hostname}
+        for server in self.nova.servers.list(detailed=False, search_opts=opts):
+            self.nova.servers.reset_state(server, 'error')
+        self.nova.services.force_down(hostname, 'nova-compute', True)
+
+
+app = Flask(__name__)
+inspector = DoctorInspectorSample()
+
+
+@app.route('/events', methods=['POST'])
+def event_posted():
+    app.logger.debug('event posted')
+    app.logger.debug('inspector = %s' % inspector)
+    app.logger.debug('received data = %s' % request.data)
+    d = json.loads(request.data)
+    hostname = d['hostname']
+    event_type = d['type']
+    if event_type == 'compute.host.down':
+        inspector.disable_compute_host(hostname)
+    return "OK"
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Doctor Sample Monitor')
+    parser.add_argument('port', metavar='PORT', type=int, nargs='?',
+                        help='a port for inspectpr')
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    app.run(port=args.port, debug=True)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/monitor.py b/tests/monitor.py
new file mode 100644
index 00000000..2a78cb66
--- /dev/null
+++ b/tests/monitor.py
@@ -0,0 +1,79 @@
+#
+# Copyright 2016 NEC Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import requests
+import socket
+import time
+
+
+# NOTE: icmp message with all zero data (checksum = 0xf7ff)
+#       see https://tools.ietf.org/html/rfc792
+ICMP_ECHO_MESSAGE = '\x08\x00\xf7\xff\x00\x00\x00\x00'
+
+
+class DoctorMonitorSample(object):
+
+    interval = 0.1  # second
+    timeout = 0.1  # second
+    event_type = "compute.host.down"
+
+    def __init__(self, args):
+        self.hostname = args.hostname
+        self.inspector = args.inspector
+        self.ip_addr = socket.gethostbyname(self.hostname)
+
+    def start_loop(self):
+        print "start ping to host %s" % self.hostname
+        sock = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                             socket.IPPROTO_ICMP)
+        sock.settimeout(self.timeout)
+        while True:
+            try:
+                sock.sendto(ICMP_ECHO_MESSAGE, (self.ip_addr, 0))
+                data = sock.recv(4096)
+            except socket.timeout:
+                print "doctor monitor detected at %s" % time.time()
+                self.report_error()
+                print "ping timeout, quit monitoring..."
+                return
+            time.sleep(self.interval)
+
+    def report_error(self):
+        payload = {"type": self.event_type, "hostname": self.hostname}
+        data = json.dumps(payload)
+        headers = {'content-type': 'application/json'}
+        requests.post(self.inspector, data=data, headers=headers)
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Doctor Sample Monitor')
+    parser.add_argument('hostname', metavar='HOSTNAME', type=str, nargs='?',
+                        help='a hostname to monitor connectivity')
+    parser.add_argument('inspector', metavar='INSPECTOR', type=str, nargs='?',
+                        help='inspector url to report error',
+                        default='http://127.0.0.1:12345/events')
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    monitor = DoctorMonitorSample(args)
+    monitor.start_loop()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/run.sh b/tests/run.sh
new file mode 100755
index 00000000..31fec615
--- /dev/null
+++ b/tests/run.sh
@@ -0,0 +1,148 @@
+#!/bin/bash -ex
+#
+# Copyright 2016 NEC Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+#branch=$(git rev-parse --abbrev-ref HEAD)
+BRANCH=master
+
+IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
+IMAGE_NAME=cirros
+IMAGE_FILE="${IMAGE_NAME}.img"
+IMAGE_FORMAT=qcow2
+VM_NAME=doctor_vm1
+VM_FLAVOR=m1.tiny
+COMPUTE_HOST='s142'
+ALARM_NAME=doctor_alarm1
+INSPECTOR_PORT=12345
+CONSUMER_PORT=12346
+
+
+download_image() {
+    [ -e "$IMAGE_FILE" ] && return 0
+    wget "$IMAGE_URL" -o "$IMAGE_FILE"
+}
+
+register_image() {
+    glance image-list | grep -q " $IMAGE_NAME " && return 0
+    glance image-create --name "$IMAGE_NAME" \
+                        --visibility public \
+                        --disk-format "$IMAGE_FORMAT" \
+                        --container-format bare \
+                        --file "$IMAGE_FILE"
+}
+
+boot_vm() {
+    nova list | grep -q " $VM_NAME " && return 0
+    nova boot --flavor "$VM_FLAVOR" \
+              --image "$IMAGE_NAME" \
+              "$VM_NAME"
+    sleep 1
+}
+
+create_alarm() {
+    ceilometer alarm-list | grep -q " $ALARM_NAME " && return 0
+    vm_id=$(nova list | grep " $VM_NAME " | awk '{print $2}')
+    ceilometer alarm-event-create --name "$ALARM_NAME" \
+        --alarm-action "http://localhost:$CONSUMER_PORT/failure" \
+        --description "VM failure" \
+        --enabled True \
+        --repeat-actions False \
+        --severity "moderate" \
+        --event-type compute.instance.update \
+        -q "traits.state=string::error; traits.instance_id=string::$vm_id"
+}
+
+start_monitor() {
+    pgrep -f "python monitor.py" && return 0
+    sudo python monitor.py "$COMPUTE_HOST" "http://127.0.0.1:$INSPECTOR_PORT/events" > monitor.log 2>&1 &
+    MONITOR_PID=$!
+}
+
+stop_monitor() {
+    pgrep -f "python monitor.py" || return 0
+    sudo kill $(pgrep -f "python monitor.py")
+    cat monitor.log
+}
+
+start_inspector() {
+    pgrep -f "python inspector.py" && return 0
+    python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
+}
+
+stop_inspector() {
+    pgrep -f "python inspector.py" || return 0
+    kill $(pgrep -f "python inspector.py")
+    cat inspector.log
+}
+
+start_consumer() {
+    pgrep -f "python consumer.py" && return 0
+    python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 &
+}
+
+stop_consumer() {
+    pgrep -f "python consumer.py" || return 0
+    kill $(pgrep -f "python consumer.py")
+    cat consumer.log
+}
+
+wait_for_vm_launch() {
+    echo "waiting for vm launch..."
+    while true
+    do
+        state=$(nova list | grep " $VM_NAME " | awk '{print $6}')
+        [[ "$state" == "ACTIVE" ]] && return 0
+        sleep 1
+    done
+}
+
+inject_failure() {
+    #FIXME
+    echo ssh $COMPUTE_HOST "ip link set eno1 down"
+}
+
+calculate_notification_time() {
+    detect=$(grep "doctor monitor detected at" monitor.log | awk '{print $5}')
+    notified=$(grep "doctor consumer notified at" consumer.log | awk '{print $5}')
+    duration=$(echo "$notified $detect" | awk '{print $1 - $2 }')
+    echo "$notified $detect" | \
+        awk '{d = $1 - $2; if (d < 1 ) print d " OK"; else print d " NG"}'
+}
+
+# TODO(r-mibu): Make sure env params are set properly for OpenStack clients
+# TODO(r-mibu): Make sure POD for doctor test is available in Pharos
+
+echo "Note: doctor/tests/run.sh has been executed, "
+echo "      but skipping this test due to lack of available test env/deployment."
+exit 0
+
+download_image
+register_image
+
+start_monitor
+start_inspector
+start_consumer
+
+boot_vm
+create_alarm
+wait_for_vm_launch
+
+sleep 60
+inject_failure
+sleep 10
+
+calculate_notification_time
+
+echo "done"