diff options
Diffstat (limited to 'opt/plugin')
15 files changed, 486 insertions, 0 deletions
diff --git a/opt/plugin/__init__.py b/opt/plugin/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/opt/plugin/__init__.py diff --git a/opt/plugin/doctor/__init__.py b/opt/plugin/doctor/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/opt/plugin/doctor/__init__.py diff --git a/opt/plugin/doctor/collector.yaml b/opt/plugin/doctor/collector.yaml new file mode 100644 index 00000000..40167588 --- /dev/null +++ b/opt/plugin/doctor/collector.yaml @@ -0,0 +1,24 @@ +# collector configuration for doctor verification job +# collect performance data from log files +collector: logfile +# transform collected data into timeline +transformer: timeline +logfiles: + - filename: doctor_consumer.log + # 2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63 + - match: doctor consumer notified at \d+(\.\d+)?$ + checkpoint: notified consumer + - filename: doctor_inspector.log + # 2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3 + # 2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3 + # 2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51 + - match: event posted at \d+(\.\d+)?$ + checkpoint: posted event + - match: doctor mark vm\(.*\) error at \d+(\.\d+)?$ + checkpoint: marked VM error + - match: doctor mark host\(.*\) down at \d+(\.\d+)?$ + checkpoint: marked host down + - filename: disable_network.log + # doctor set host down at 1482894965.164096803 + - match: doctor set host down at \d+(\.\d+)?$ + - checkpoint: set host down diff --git a/opt/plugin/doctor/sample/__init__.py b/opt/plugin/doctor/sample/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/opt/plugin/doctor/sample/__init__.py diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log new file mode 100755 index 00000000..8c577b59 --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log @@ -0,0 +1,4 @@ + * Running on http://0.0.0.0:12346/ (Press CTRL+C to quit) +2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63 +2016-12-28 03:16:05,630 consumer.py 27 INFO received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"} +127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /failure HTTP/1.1" 200 - diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log new file mode 100755 index 00000000..1b488491 --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log @@ -0,0 +1,11 @@ +++ sudo ip a +++ awk '/ 192.0.2.5\//{print $7}' ++ dev=eth0 ++ sleep 1 +++ date +%s.%N ++ echo 'doctor set host down at' 1482894965.164096803 +doctor set host down at 1482894965.164096803 ++ sudo ip link set eth0 down ++ sleep 180 ++ sudo ip link set eth0 up ++ sleep 1 diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log new file mode 100755 index 00000000..e040e809 --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log @@ -0,0 +1,2 @@ +2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63 +2016-12-28 03:16:05,630 consumer.py 27 INFO received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"} diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log new file mode 100755 index 00000000..e2fd9477 --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log @@ -0,0 +1,6 @@ +2016-12-28 03:15:01,868 inspector.py 50 DEBUG get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1> +2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3 +2016-12-28 03:16:05,299 inspector.py 77 INFO inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90> +2016-12-28 03:16:05,299 inspector.py 78 INFO received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"} +2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3 +2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51 diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log new file mode 100755 index 00000000..a4c6ec8f --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log @@ -0,0 +1,3 @@ +2016-12-28 03:15:00,890 monitor.py 65 DEBUG start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5) +2016-12-28 03:16:05,292 monitor.py 74 INFO doctor monitor detected at 1482894965.29 +2016-12-28 03:16:05,960 monitor.py 76 INFO ping timeout, quit monitoring... diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log new file mode 100755 index 00000000..2933ad61 --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log @@ -0,0 +1,421 @@ ++ IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img ++ IMAGE_NAME=cirros ++ IMAGE_FILE=cirros.img ++ IMAGE_FORMAT=qcow2 ++ VM_NAME=doctor_vm1 ++ VM_FLAVOR=m1.tiny ++ ALARM_NAME=doctor_alarm1 ++ INSPECTOR_PORT=12345 ++ CONSUMER_PORT=12346 ++ DOCTOR_USER=doctor ++ DOCTOR_PW=doctor ++ DOCTOR_PROJECT=doctor ++ DOCTOR_ROLE=admin ++++ dirname ./run.sh +++ cd . +++ pwd ++ TOP_DIR=/home/opnfv/repos/doctor/tests ++ ssh_opts='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' ++ as_doctor_user='--os-username doctor --os-password doctor + --os-tenant-name doctor' ++ echo 'Note: doctor/tests/run.sh has been executed.' +Note: doctor/tests/run.sh has been executed. ++ trap cleanup EXIT ++ source /home/opnfv/repos/doctor/tests/functions-common ++ source /home/opnfv/repos/doctor/tests/lib/installer +++ INSTALLER_TYPE=apex +++ INSTALLER_IP=192.168.122.70 +++ ssh_opts_cpu='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' ++ source /home/opnfv/repos/doctor/tests/lib/inspector +++ INSPECTOR_TYPE=sample ++ setup_installer ++ is_installer_supported apex ++ local installer=apex ++ [[ -f /home/opnfv/repos/doctor/tests/lib/installers/apex ]] ++ source /home/opnfv/repos/doctor/tests/lib/installers/apex ++ is_set INSTALLER_IP ++ local 'var=$INSTALLER_IP' ++ eval '[ -n "$INSTALLER_IP" ]' +++ '[' -n 192.168.122.70 ']' ++ installer_get_ssh_keys ++ sudo scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@192.168.122.70:/home/stack/.ssh/id_rsa instack_key +Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
+++ whoami +++ whoami ++ sudo chown root:root instack_key ++ chmod 400 instack_key ++ ssh_opts_cpu+=' -i instack_key' ++ installer_apply_patches ++ return ++ echo 'preparing VM image...' +preparing VM image... ++ download_image ++ use_existing_image=false ++ openstack image list ++ grep -q ' cirros ' ++ use_existing_image=true ++ [[ true == false ]] ++ register_image ++ openstack image list ++ grep -q ' cirros ' ++ return 0 ++ echo 'creating test user...' +creating test user... ++ create_test_user ++ openstack project list ++ grep -q ' doctor ' ++ openstack project create doctor ++-------------+----------------------------------+ +| Field | Value | ++-------------+----------------------------------+ +| description | None | +| enabled | True | +| id | dffffadad42d41d5a618570f52e3305f | +| name | doctor | ++-------------+----------------------------------+ ++ openstack user list ++ grep -q ' doctor ' ++ openstack user create doctor --password doctor --project doctor ++------------+----------------------------------+ +| Field | Value | ++------------+----------------------------------+ +| email | None | +| enabled | True | +| id | c844f5397a924d908f1246515e4843e6 | +| name | doctor | +| project_id | dffffadad42d41d5a618570f52e3305f | +| username | doctor | ++------------+----------------------------------+ ++ openstack user role list doctor --project doctor ++ grep -q ' admin ' ++ openstack role add admin --user doctor --project doctor ++-----------+----------------------------------+ +| Field | Value | ++-----------+----------------------------------+ +| domain_id | None | +| id | 4b0d10e07705433c865958471d7c4ff5 | +| name | admin | ++-----------+----------------------------------+ ++ echo 'creating VM...' +creating VM... ++ boot_vm ++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list ++ grep -q ' doctor_vm1 ' ++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server create --flavor m1.tiny --image cirros doctor_vm1 ++--------------------------------------+------------------------------------------------+ +| Field | Value | ++--------------------------------------+------------------------------------------------+ +| OS-DCF:diskConfig | MANUAL | +| OS-EXT-AZ:availability_zone | | +| OS-EXT-SRV-ATTR:host | None | +| OS-EXT-SRV-ATTR:hypervisor_hostname | None | +| OS-EXT-SRV-ATTR:instance_name | | +| OS-EXT-STS:power_state | 0 | +| OS-EXT-STS:task_state | scheduling | +| OS-EXT-STS:vm_state | building | +| OS-SRV-USG:launched_at | None | +| OS-SRV-USG:terminated_at | None | +| accessIPv4 | | +| accessIPv6 | | +| addresses | | +| adminPass | jL7t3S3DUrCX | +| config_drive | | +| created | 2016-12-28T03:14:37Z | +| flavor | m1.tiny (7f924f9f-2fa0-40d4-95fb-53079307ec66) | +| hostId | | +| id | 51c0739a-0b87-4dd7-8d22-ff7fd8af04bb | +| image | cirros (3071d9bf-7be5-4cb1-a706-428daa446ec3) | +| key_name | None | +| name | doctor_vm1 | +| os-extended-volumes:volumes_attached | [] | +| progress | 0 | +| project_id | dffffadad42d41d5a618570f52e3305f | +| properties | | +| security_groups | [{u'name': u'default'}] | +| status | BUILD | +| updated | 2016-12-28T03:14:37Z | +| user_id | c844f5397a924d908f1246515e4843e6 | ++--------------------------------------+------------------------------------------------+ ++ sleep 1 ++ wait_for_vm_launch ++ echo 'waiting for vm launch...' +waiting for vm launch... ++ count=0 ++ [[ 0 -lt 60 ]] +++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list +++ grep ' doctor_vm1 ' +++ awk '{print $6}' ++ state=BUILD ++ [[ BUILD == \A\C\T\I\V\E ]] ++ [[ BUILD == \E\R\R\O\R ]] ++ count=1 ++ sleep 1 ++ [[ 1 -lt 60 ]] +++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list +++ grep ' doctor_vm1 ' +++ awk '{print $6}' ++ state=ACTIVE ++ [[ ACTIVE == \A\C\T\I\V\E ]] ++ sleep 5 ++ return 0 ++ echo 'get computer host info...' +get computer host info... ++ get_compute_host_info +++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server show doctor_vm1 +++ grep OS-EXT-SRV-ATTR:host +++ awk '{ print $4 }' ++ COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1 ++ compute_host_in_undercloud=overcloud-novacompute-1 ++ die_if_not_set 45 COMPUTE_HOST 'Failed to get compute hostname' ++ local exitcode=0 ++ local xtrace +++ set +o +++ grep xtrace ++ xtrace='set -o xtrace' ++ set +o xtrace ++ is_installer apex ++ local installer=apex ++ [[ apex == apex ]] ++ COMPUTE_USER=heat-admin +++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc; nova show overcloud-novacompute-1 | awk '\''/ ctlplane network /{print $5}'\''' +Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
++ COMPUTE_IP=192.0.2.5 ++ die_if_not_set 63 COMPUTE_IP 'Could not resolve overcloud-novacompute-1.ool-virtual1. Either manually set COMPUTE_IP or enable DNS resolution.' ++ local exitcode=0 ++ local xtrace +++ set +o +++ grep xtrace ++ xtrace='set -o xtrace' ++ set +o xtrace ++ echo COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1 +COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1 ++ echo COMPUTE_IP=192.0.2.5 +COMPUTE_IP=192.0.2.5 ++ ping -c 1 192.0.2.5 +PING 192.0.2.5 (192.0.2.5) 56(84) bytes of data. +64 bytes from 192.0.2.5: icmp_seq=1 ttl=63 time=0.275 ms + +--- 192.0.2.5 ping statistics --- +1 packets transmitted, 1 received, 0% packet loss, time 0ms +rtt min/avg/max/mdev = 0.275/0.275/0.275/0.000 ms ++ [[ 0 -ne 0 ]] ++ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 exit +Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ [[ 0 -ne 0 ]] ++ echo 'creating alarm...' +creating alarm... ++ create_alarm ++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list ++ grep -q ' doctor_alarm1 ' +++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list +++ grep ' doctor_vm1 ' +++ awk '{print $2}' ++ vm_id=51c0739a-0b87-4dd7-8d22-ff7fd8af04bb ++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-event-create --name doctor_alarm1 --alarm-action http://localhost:12346/failure --description 'VM failure' --enabled True --repeat-actions False --severity moderate --event-type compute.instance.update -q 'traits.state=string::error; traits.instance_id=string::51c0739a-0b87-4dd7-8d22-ff7fd8af04bb' ++---------------------------+----------------------------------------------------------------------+ +| Property | Value | ++---------------------------+----------------------------------------------------------------------+ +| alarm_actions | ["http://localhost:12346/failure"] | +| alarm_id | d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 | +| description | VM failure | +| enabled | True | +| event_type | compute.instance.update | +| insufficient_data_actions | [] | +| name | doctor_alarm1 | +| ok_actions | [] | +| project_id | dffffadad42d41d5a618570f52e3305f | +| query | [{"field": "traits.state", "type": "string", "value": "error", "op": | +| | "eq"}, {"field": "traits.instance_id", "type": "string", "value": | +| | "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb", "op": "eq"}] | +| repeat_actions | False | +| severity | moderate | +| state | insufficient data | +| type | event | +| user_id | c844f5397a924d908f1246515e4843e6 | ++---------------------------+----------------------------------------------------------------------+ ++ echo 'starting doctor sample components...' +starting doctor sample components... ++ start_inspector ++ is_inspector_supported sample ++ local inspector=sample ++ [[ -f /home/opnfv/repos/doctor/tests/lib/inspectors/sample ]] ++ source /home/opnfv/repos/doctor/tests/lib/inspectors/sample ++ start_inspector_sample ++ pgrep -f 'python inspector.py' ++ start_monitor ++ pgrep -f 'python monitor.py' ++ python inspector.py 12345 ++ start_consumer ++ sudo -E python monitor.py overcloud-novacompute-1.ool-virtual1 192.0.2.5 sample http://127.0.0.1:12345/events ++ pgrep -f 'python consumer.py' ++ is_installer local ++ local installer=local ++ python consumer.py 12346 ++ [[ local == apex ]] ++ is_installer apex ++ local installer=apex ++ [[ apex == apex ]] +++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc; nova list | grep '\'' overcloud-controller-[0-9] '\'' | sed -e '\''s/^.*ctlplane=//'\'' -e '\''s/ *|$//'\''' +Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
++ CONTROLLER_IPS='192.0.2.6 +192.0.2.8 +192.0.2.9' ++ die_if_not_set 185 CONTROLLER_IPS 'Could not get CONTROLLER_IPS.' ++ local exitcode=0 ++ local xtrace +++ set +o +++ grep xtrace ++ xtrace='set -o xtrace' ++ set +o xtrace ++ for ip in '$CONTROLLER_IPS' ++ forward_rule='-R 12346:localhost:12346' ++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600' ++ for ip in '$CONTROLLER_IPS' ++ forward_rule='-R 12346:localhost:12346' ++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600' ++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600 ++ for ip in '$CONTROLLER_IPS' ++ forward_rule='-R 12346:localhost:12346' ++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600' ++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600 ++ sleep 60 ++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600 ++ echo 'injecting host failure...' +injecting host failure... ++ inject_failure ++ echo 'disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins...' +disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins... ++ cat ++ sed -i -e s/@COMPUTE_IP@/192.0.2.5/ disable_network.sh ++ chmod +x disable_network.sh ++ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key disable_network.sh heat-admin@192.0.2.5: +Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 'nohup ./disable_network.sh > disable_network.log 2>&1 &' +Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ sleep 60 ++ check_host_status '(DOWN|UNKNOWN)' ++ expected_state='(DOWN|UNKNOWN)' +++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1 +++ grep host_status ++ host_status_line='| host_status | DOWN |' +++ echo '|' host_status '|' DOWN '|' +++ awk '{print $4}' ++ host_status=DOWN ++ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1' ++ local exitcode=0 ++ local xtrace +++ set +o +++ grep xtrace ++ xtrace='set -o xtrace' ++ set +o xtrace ++ [[ (DOWN|UNKNOWN) =~ DOWN ]] ++ echo 'doctor_vm1 showing host_status: DOWN' +doctor_vm1 showing host_status: DOWN ++ calculate_notification_time +++ grep 'doctor monitor detected at' monitor.log +++ awk '{print $10}' ++ detected=1482894965.29 +++ grep 'doctor consumer notified at' consumer.log +++ awk '{print $10}' ++ notified=1482894965.63 ++ grep -q 'doctor consumer notified at' consumer.log ++ [[ PROFILER == \p\o\c ]] ++ echo '1482894965.63 1482894965.29' ++ awk '{ + d = $1 - $2; + if (d < 1 && d > 0) { print d " OK"; exit 0 } + else { print d " NG"; exit 1 } + }' +0.34 OK ++ echo done +done ++ cleanup ++ set +e ++ echo cleanup... +cleanup... ++ stop_monitor ++ pgrep -f 'python monitor.py' ++ return 0 ++ stop_inspector ++ stop_inspector_sample ++ pgrep -f 'python inspector.py' +377 +++ pgrep -f 'python inspector.py' ++ kill 377 ++ stop_consumer ++ pgrep -f 'python consumer.py' +381 +++ pgrep -f 'python consumer.py' ++ kill 381 ++ is_installer local ++ local installer=local ++ [[ local == apex ]] ++ for ip in '$CONTROLLER_IPS' ++ forward_rule='-R 12346:localhost:12346' ++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600' +++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600' ++ kill 398 ++ for ip in '$CONTROLLER_IPS' ++ forward_rule='-R 12346:localhost:12346' ++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600' +./run.sh: line 206: 377 Terminated python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 +./run.sh: line 206: 381 Terminated python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 +++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600' ++ kill 399 ++ for ip in '$CONTROLLER_IPS' ++ forward_rule='-R 12346:localhost:12346' ++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600' +++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600' ++ kill 400 ++ echo 'waiting disabled compute host back to be enabled...' +waiting disabled compute host back to be enabled... ++ python ./nova_force_down.py overcloud-novacompute-1.ool-virtual1 --unset +<Response [200]> ++ sleep 240 ++ check_host_status UP ++ expected_state=UP +++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1 +++ grep host_status ++ host_status_line='| host_status | UP |' +++ echo '|' host_status '|' UP '|' +++ awk '{print $4}' ++ host_status=UP ++ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1' ++ local exitcode=0 ++ local xtrace +++ set +o +++ grep xtrace ++ xtrace='set -o xtrace' ++ set +o xtrace ++ [[ UP =~ UP ]] ++ echo 'doctor_vm1 showing host_status: UP' +doctor_vm1 showing host_status: UP ++ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5:disable_network.log . +Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list ++ grep -q ' doctor_vm1 ' ++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server delete doctor_vm1 ++ sleep 1 +++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list +++ grep ' doctor_alarm1 ' +++ awk '{print $2}' ++ alarm_id=d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 ++ sleep 1 ++ '[' -n d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 ']' ++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-delete d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 ++ sleep 1 +++ openstack image list +++ grep ' cirros ' +++ awk '{print $2}' ++ image_id=3071d9bf-7be5-4cb1-a706-428daa446ec3 ++ sleep 1 ++ [[ true == false ]] ++ openstack role remove admin --user doctor --project doctor ++ openstack project delete doctor ++ openstack user delete doctor ++ cleanup_installer ++ cleanup_installer_apex ++ return ++ cleanup_inspector ++ cleanup_inspector_sample ++ return diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log new file mode 100755 index 00000000..0d07de02 --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log @@ -0,0 +1,9 @@ +2016-12-28 03:15:01,868 inspector.py 50 DEBUG get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1> + * Running on http://127.0.0.1:12345/ (Press CTRL+C to quit) +2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3 +2016-12-28 03:16:05,299 inspector.py 77 INFO inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90> +2016-12-28 03:16:05,299 inspector.py 78 INFO received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"} +2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3 +2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51 +<Response [200]> +127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /events HTTP/1.1" 200 - diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log new file mode 100755 index 00000000..a4c6ec8f --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log @@ -0,0 +1,3 @@ +2016-12-28 03:15:00,890 monitor.py 65 DEBUG start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5) +2016-12-28 03:16:05,292 monitor.py 74 INFO doctor monitor detected at 1482894965.29 +2016-12-28 03:16:05,960 monitor.py 76 INFO ping timeout, quit monitoring... diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log new file mode 100755 index 00000000..70cc8945 --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log @@ -0,0 +1 @@ +Warning: Permanently added '192.0.2.6' (ECDSA) to the list of known hosts.
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log new file mode 100755 index 00000000..54222d2a --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log @@ -0,0 +1 @@ +Warning: Permanently added '192.0.2.8' (ECDSA) to the list of known hosts.
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log new file mode 100755 index 00000000..2660035b --- /dev/null +++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log @@ -0,0 +1 @@ +Warning: Permanently added '192.0.2.9' (ECDSA) to the list of known hosts.
|