summaryrefslogtreecommitdiffstats
path: root/opt/plugin/doctor
diff options
context:
space:
mode:
authorYujun Zhang <zhang.yujunz@zte.com.cn>2017-01-17 21:58:44 +0800
committerYujun Zhang <zhang.yujunz@zte.com.cn>2017-01-23 12:17:27 +0800
commitd728b82cbff33ab7658e8aa0f2e846ae3afe3606 (patch)
tree491c74d47a8ac363a88310e026c873f5f1a38e49 /opt/plugin/doctor
parentb659a9296223b7f63a89bedbc0dad281117de936 (diff)
Code refactoring about doctor QPI
- Move sample log files to test data - Move configuration from plugin to plan Change-Id: Iaaf36ab3ce29c2eafe2d75f3524e7a302cc3836d Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>
Diffstat (limited to 'opt/plugin/doctor')
-rw-r--r--opt/plugin/doctor/__init__.py0
-rw-r--r--opt/plugin/doctor/collector.yaml24
-rw-r--r--opt/plugin/doctor/sample/__init__.py0
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log4
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log11
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log2
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log6
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log3
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log421
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log9
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log3
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log1
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log1
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log1
14 files changed, 0 insertions, 486 deletions
diff --git a/opt/plugin/doctor/__init__.py b/opt/plugin/doctor/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/opt/plugin/doctor/__init__.py
+++ /dev/null
diff --git a/opt/plugin/doctor/collector.yaml b/opt/plugin/doctor/collector.yaml
deleted file mode 100644
index 40167588..00000000
--- a/opt/plugin/doctor/collector.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-# collector configuration for doctor verification job
-# collect performance data from log files
-collector: logfile
-# transform collected data into timeline
-transformer: timeline
-logfiles:
- - filename: doctor_consumer.log
- # 2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63
- - match: doctor consumer notified at \d+(\.\d+)?$
- checkpoint: notified consumer
- - filename: doctor_inspector.log
- # 2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3
- # 2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
- # 2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
- - match: event posted at \d+(\.\d+)?$
- checkpoint: posted event
- - match: doctor mark vm\(.*\) error at \d+(\.\d+)?$
- checkpoint: marked VM error
- - match: doctor mark host\(.*\) down at \d+(\.\d+)?$
- checkpoint: marked host down
- - filename: disable_network.log
- # doctor set host down at 1482894965.164096803
- - match: doctor set host down at \d+(\.\d+)?$
- - checkpoint: set host down
diff --git a/opt/plugin/doctor/sample/__init__.py b/opt/plugin/doctor/sample/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/opt/plugin/doctor/sample/__init__.py
+++ /dev/null
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log
deleted file mode 100755
index 8c577b59..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log
+++ /dev/null
@@ -1,4 +0,0 @@
- * Running on http://0.0.0.0:12346/ (Press CTRL+C to quit)
-2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63
-2016-12-28 03:16:05,630 consumer.py 27 INFO received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"}
-127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /failure HTTP/1.1" 200 -
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log
deleted file mode 100755
index 1b488491..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log
+++ /dev/null
@@ -1,11 +0,0 @@
-++ sudo ip a
-++ awk '/ 192.0.2.5\//{print $7}'
-+ dev=eth0
-+ sleep 1
-++ date +%s.%N
-+ echo 'doctor set host down at' 1482894965.164096803
-doctor set host down at 1482894965.164096803
-+ sudo ip link set eth0 down
-+ sleep 180
-+ sudo ip link set eth0 up
-+ sleep 1
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log
deleted file mode 100755
index e040e809..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log
+++ /dev/null
@@ -1,2 +0,0 @@
-2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63
-2016-12-28 03:16:05,630 consumer.py 27 INFO received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"}
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log
deleted file mode 100755
index e2fd9477..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log
+++ /dev/null
@@ -1,6 +0,0 @@
-2016-12-28 03:15:01,868 inspector.py 50 DEBUG get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1>
-2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3
-2016-12-28 03:16:05,299 inspector.py 77 INFO inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90>
-2016-12-28 03:16:05,299 inspector.py 78 INFO received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"}
-2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
-2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log
deleted file mode 100755
index a4c6ec8f..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log
+++ /dev/null
@@ -1,3 +0,0 @@
-2016-12-28 03:15:00,890 monitor.py 65 DEBUG start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5)
-2016-12-28 03:16:05,292 monitor.py 74 INFO doctor monitor detected at 1482894965.29
-2016-12-28 03:16:05,960 monitor.py 76 INFO ping timeout, quit monitoring...
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log
deleted file mode 100755
index 2933ad61..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log
+++ /dev/null
@@ -1,421 +0,0 @@
-+ IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
-+ IMAGE_NAME=cirros
-+ IMAGE_FILE=cirros.img
-+ IMAGE_FORMAT=qcow2
-+ VM_NAME=doctor_vm1
-+ VM_FLAVOR=m1.tiny
-+ ALARM_NAME=doctor_alarm1
-+ INSPECTOR_PORT=12345
-+ CONSUMER_PORT=12346
-+ DOCTOR_USER=doctor
-+ DOCTOR_PW=doctor
-+ DOCTOR_PROJECT=doctor
-+ DOCTOR_ROLE=admin
-+++ dirname ./run.sh
-++ cd .
-++ pwd
-+ TOP_DIR=/home/opnfv/repos/doctor/tests
-+ ssh_opts='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-+ as_doctor_user='--os-username doctor --os-password doctor
- --os-tenant-name doctor'
-+ echo 'Note: doctor/tests/run.sh has been executed.'
-Note: doctor/tests/run.sh has been executed.
-+ trap cleanup EXIT
-+ source /home/opnfv/repos/doctor/tests/functions-common
-+ source /home/opnfv/repos/doctor/tests/lib/installer
-++ INSTALLER_TYPE=apex
-++ INSTALLER_IP=192.168.122.70
-++ ssh_opts_cpu='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-+ source /home/opnfv/repos/doctor/tests/lib/inspector
-++ INSPECTOR_TYPE=sample
-+ setup_installer
-+ is_installer_supported apex
-+ local installer=apex
-+ [[ -f /home/opnfv/repos/doctor/tests/lib/installers/apex ]]
-+ source /home/opnfv/repos/doctor/tests/lib/installers/apex
-+ is_set INSTALLER_IP
-+ local 'var=$INSTALLER_IP'
-+ eval '[ -n "$INSTALLER_IP" ]'
-++ '[' -n 192.168.122.70 ']'
-+ installer_get_ssh_keys
-+ sudo scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@192.168.122.70:/home/stack/.ssh/id_rsa instack_key
-Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
-++ whoami
-++ whoami
-+ sudo chown root:root instack_key
-+ chmod 400 instack_key
-+ ssh_opts_cpu+=' -i instack_key'
-+ installer_apply_patches
-+ return
-+ echo 'preparing VM image...'
-preparing VM image...
-+ download_image
-+ use_existing_image=false
-+ openstack image list
-+ grep -q ' cirros '
-+ use_existing_image=true
-+ [[ true == false ]]
-+ register_image
-+ openstack image list
-+ grep -q ' cirros '
-+ return 0
-+ echo 'creating test user...'
-creating test user...
-+ create_test_user
-+ openstack project list
-+ grep -q ' doctor '
-+ openstack project create doctor
-+-------------+----------------------------------+
-| Field | Value |
-+-------------+----------------------------------+
-| description | None |
-| enabled | True |
-| id | dffffadad42d41d5a618570f52e3305f |
-| name | doctor |
-+-------------+----------------------------------+
-+ openstack user list
-+ grep -q ' doctor '
-+ openstack user create doctor --password doctor --project doctor
-+------------+----------------------------------+
-| Field | Value |
-+------------+----------------------------------+
-| email | None |
-| enabled | True |
-| id | c844f5397a924d908f1246515e4843e6 |
-| name | doctor |
-| project_id | dffffadad42d41d5a618570f52e3305f |
-| username | doctor |
-+------------+----------------------------------+
-+ openstack user role list doctor --project doctor
-+ grep -q ' admin '
-+ openstack role add admin --user doctor --project doctor
-+-----------+----------------------------------+
-| Field | Value |
-+-----------+----------------------------------+
-| domain_id | None |
-| id | 4b0d10e07705433c865958471d7c4ff5 |
-| name | admin |
-+-----------+----------------------------------+
-+ echo 'creating VM...'
-creating VM...
-+ boot_vm
-+ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
-+ grep -q ' doctor_vm1 '
-+ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server create --flavor m1.tiny --image cirros doctor_vm1
-+--------------------------------------+------------------------------------------------+
-| Field | Value |
-+--------------------------------------+------------------------------------------------+
-| OS-DCF:diskConfig | MANUAL |
-| OS-EXT-AZ:availability_zone | |
-| OS-EXT-SRV-ATTR:host | None |
-| OS-EXT-SRV-ATTR:hypervisor_hostname | None |
-| OS-EXT-SRV-ATTR:instance_name | |
-| OS-EXT-STS:power_state | 0 |
-| OS-EXT-STS:task_state | scheduling |
-| OS-EXT-STS:vm_state | building |
-| OS-SRV-USG:launched_at | None |
-| OS-SRV-USG:terminated_at | None |
-| accessIPv4 | |
-| accessIPv6 | |
-| addresses | |
-| adminPass | jL7t3S3DUrCX |
-| config_drive | |
-| created | 2016-12-28T03:14:37Z |
-| flavor | m1.tiny (7f924f9f-2fa0-40d4-95fb-53079307ec66) |
-| hostId | |
-| id | 51c0739a-0b87-4dd7-8d22-ff7fd8af04bb |
-| image | cirros (3071d9bf-7be5-4cb1-a706-428daa446ec3) |
-| key_name | None |
-| name | doctor_vm1 |
-| os-extended-volumes:volumes_attached | [] |
-| progress | 0 |
-| project_id | dffffadad42d41d5a618570f52e3305f |
-| properties | |
-| security_groups | [{u'name': u'default'}] |
-| status | BUILD |
-| updated | 2016-12-28T03:14:37Z |
-| user_id | c844f5397a924d908f1246515e4843e6 |
-+--------------------------------------+------------------------------------------------+
-+ sleep 1
-+ wait_for_vm_launch
-+ echo 'waiting for vm launch...'
-waiting for vm launch...
-+ count=0
-+ [[ 0 -lt 60 ]]
-++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
-++ grep ' doctor_vm1 '
-++ awk '{print $6}'
-+ state=BUILD
-+ [[ BUILD == \A\C\T\I\V\E ]]
-+ [[ BUILD == \E\R\R\O\R ]]
-+ count=1
-+ sleep 1
-+ [[ 1 -lt 60 ]]
-++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
-++ grep ' doctor_vm1 '
-++ awk '{print $6}'
-+ state=ACTIVE
-+ [[ ACTIVE == \A\C\T\I\V\E ]]
-+ sleep 5
-+ return 0
-+ echo 'get computer host info...'
-get computer host info...
-+ get_compute_host_info
-++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server show doctor_vm1
-++ grep OS-EXT-SRV-ATTR:host
-++ awk '{ print $4 }'
-+ COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
-+ compute_host_in_undercloud=overcloud-novacompute-1
-+ die_if_not_set 45 COMPUTE_HOST 'Failed to get compute hostname'
-+ local exitcode=0
-+ local xtrace
-++ set +o
-++ grep xtrace
-+ xtrace='set -o xtrace'
-+ set +o xtrace
-+ is_installer apex
-+ local installer=apex
-+ [[ apex == apex ]]
-+ COMPUTE_USER=heat-admin
-++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc; nova show overcloud-novacompute-1 | awk '\''/ ctlplane network /{print $5}'\'''
-Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
-+ COMPUTE_IP=192.0.2.5
-+ die_if_not_set 63 COMPUTE_IP 'Could not resolve overcloud-novacompute-1.ool-virtual1. Either manually set COMPUTE_IP or enable DNS resolution.'
-+ local exitcode=0
-+ local xtrace
-++ set +o
-++ grep xtrace
-+ xtrace='set -o xtrace'
-+ set +o xtrace
-+ echo COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
-COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
-+ echo COMPUTE_IP=192.0.2.5
-COMPUTE_IP=192.0.2.5
-+ ping -c 1 192.0.2.5
-PING 192.0.2.5 (192.0.2.5) 56(84) bytes of data.
-64 bytes from 192.0.2.5: icmp_seq=1 ttl=63 time=0.275 ms
-
---- 192.0.2.5 ping statistics ---
-1 packets transmitted, 1 received, 0% packet loss, time 0ms
-rtt min/avg/max/mdev = 0.275/0.275/0.275/0.000 ms
-+ [[ 0 -ne 0 ]]
-+ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 exit
-Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
-+ [[ 0 -ne 0 ]]
-+ echo 'creating alarm...'
-creating alarm...
-+ create_alarm
-+ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list
-+ grep -q ' doctor_alarm1 '
-++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
-++ grep ' doctor_vm1 '
-++ awk '{print $2}'
-+ vm_id=51c0739a-0b87-4dd7-8d22-ff7fd8af04bb
-+ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-event-create --name doctor_alarm1 --alarm-action http://localhost:12346/failure --description 'VM failure' --enabled True --repeat-actions False --severity moderate --event-type compute.instance.update -q 'traits.state=string::error; traits.instance_id=string::51c0739a-0b87-4dd7-8d22-ff7fd8af04bb'
-+---------------------------+----------------------------------------------------------------------+
-| Property | Value |
-+---------------------------+----------------------------------------------------------------------+
-| alarm_actions | ["http://localhost:12346/failure"] |
-| alarm_id | d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 |
-| description | VM failure |
-| enabled | True |
-| event_type | compute.instance.update |
-| insufficient_data_actions | [] |
-| name | doctor_alarm1 |
-| ok_actions | [] |
-| project_id | dffffadad42d41d5a618570f52e3305f |
-| query | [{"field": "traits.state", "type": "string", "value": "error", "op": |
-| | "eq"}, {"field": "traits.instance_id", "type": "string", "value": |
-| | "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb", "op": "eq"}] |
-| repeat_actions | False |
-| severity | moderate |
-| state | insufficient data |
-| type | event |
-| user_id | c844f5397a924d908f1246515e4843e6 |
-+---------------------------+----------------------------------------------------------------------+
-+ echo 'starting doctor sample components...'
-starting doctor sample components...
-+ start_inspector
-+ is_inspector_supported sample
-+ local inspector=sample
-+ [[ -f /home/opnfv/repos/doctor/tests/lib/inspectors/sample ]]
-+ source /home/opnfv/repos/doctor/tests/lib/inspectors/sample
-+ start_inspector_sample
-+ pgrep -f 'python inspector.py'
-+ start_monitor
-+ pgrep -f 'python monitor.py'
-+ python inspector.py 12345
-+ start_consumer
-+ sudo -E python monitor.py overcloud-novacompute-1.ool-virtual1 192.0.2.5 sample http://127.0.0.1:12345/events
-+ pgrep -f 'python consumer.py'
-+ is_installer local
-+ local installer=local
-+ python consumer.py 12346
-+ [[ local == apex ]]
-+ is_installer apex
-+ local installer=apex
-+ [[ apex == apex ]]
-++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc; nova list | grep '\'' overcloud-controller-[0-9] '\'' | sed -e '\''s/^.*ctlplane=//'\'' -e '\''s/ *|$//'\'''
-Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
-+ CONTROLLER_IPS='192.0.2.6
-192.0.2.8
-192.0.2.9'
-+ die_if_not_set 185 CONTROLLER_IPS 'Could not get CONTROLLER_IPS.'
-+ local exitcode=0
-+ local xtrace
-++ set +o
-++ grep xtrace
-+ xtrace='set -o xtrace'
-+ set +o xtrace
-+ for ip in '$CONTROLLER_IPS'
-+ forward_rule='-R 12346:localhost:12346'
-+ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
-+ for ip in '$CONTROLLER_IPS'
-+ forward_rule='-R 12346:localhost:12346'
-+ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
-+ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600
-+ for ip in '$CONTROLLER_IPS'
-+ forward_rule='-R 12346:localhost:12346'
-+ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
-+ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600
-+ sleep 60
-+ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600
-+ echo 'injecting host failure...'
-injecting host failure...
-+ inject_failure
-+ echo 'disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins...'
-disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins...
-+ cat
-+ sed -i -e s/@COMPUTE_IP@/192.0.2.5/ disable_network.sh
-+ chmod +x disable_network.sh
-+ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key disable_network.sh heat-admin@192.0.2.5:
-Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
-+ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
-Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
-+ sleep 60
-+ check_host_status '(DOWN|UNKNOWN)'
-+ expected_state='(DOWN|UNKNOWN)'
-++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1
-++ grep host_status
-+ host_status_line='| host_status | DOWN |'
-++ echo '|' host_status '|' DOWN '|'
-++ awk '{print $4}'
-+ host_status=DOWN
-+ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1'
-+ local exitcode=0
-+ local xtrace
-++ set +o
-++ grep xtrace
-+ xtrace='set -o xtrace'
-+ set +o xtrace
-+ [[ (DOWN|UNKNOWN) =~ DOWN ]]
-+ echo 'doctor_vm1 showing host_status: DOWN'
-doctor_vm1 showing host_status: DOWN
-+ calculate_notification_time
-++ grep 'doctor monitor detected at' monitor.log
-++ awk '{print $10}'
-+ detected=1482894965.29
-++ grep 'doctor consumer notified at' consumer.log
-++ awk '{print $10}'
-+ notified=1482894965.63
-+ grep -q 'doctor consumer notified at' consumer.log
-+ [[ PROFILER == \p\o\c ]]
-+ echo '1482894965.63 1482894965.29'
-+ awk '{
- d = $1 - $2;
- if (d < 1 && d > 0) { print d " OK"; exit 0 }
- else { print d " NG"; exit 1 }
- }'
-0.34 OK
-+ echo done
-done
-+ cleanup
-+ set +e
-+ echo cleanup...
-cleanup...
-+ stop_monitor
-+ pgrep -f 'python monitor.py'
-+ return 0
-+ stop_inspector
-+ stop_inspector_sample
-+ pgrep -f 'python inspector.py'
-377
-++ pgrep -f 'python inspector.py'
-+ kill 377
-+ stop_consumer
-+ pgrep -f 'python consumer.py'
-381
-++ pgrep -f 'python consumer.py'
-+ kill 381
-+ is_installer local
-+ local installer=local
-+ [[ local == apex ]]
-+ for ip in '$CONTROLLER_IPS'
-+ forward_rule='-R 12346:localhost:12346'
-+ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
-++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
-+ kill 398
-+ for ip in '$CONTROLLER_IPS'
-+ forward_rule='-R 12346:localhost:12346'
-+ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
-./run.sh: line 206: 377 Terminated python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1
-./run.sh: line 206: 381 Terminated python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1
-++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
-+ kill 399
-+ for ip in '$CONTROLLER_IPS'
-+ forward_rule='-R 12346:localhost:12346'
-+ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
-++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
-+ kill 400
-+ echo 'waiting disabled compute host back to be enabled...'
-waiting disabled compute host back to be enabled...
-+ python ./nova_force_down.py overcloud-novacompute-1.ool-virtual1 --unset
-<Response [200]>
-+ sleep 240
-+ check_host_status UP
-+ expected_state=UP
-++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1
-++ grep host_status
-+ host_status_line='| host_status | UP |'
-++ echo '|' host_status '|' UP '|'
-++ awk '{print $4}'
-+ host_status=UP
-+ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1'
-+ local exitcode=0
-+ local xtrace
-++ set +o
-++ grep xtrace
-+ xtrace='set -o xtrace'
-+ set +o xtrace
-+ [[ UP =~ UP ]]
-+ echo 'doctor_vm1 showing host_status: UP'
-doctor_vm1 showing host_status: UP
-+ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5:disable_network.log .
-Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
-+ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
-+ grep -q ' doctor_vm1 '
-+ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server delete doctor_vm1
-+ sleep 1
-++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list
-++ grep ' doctor_alarm1 '
-++ awk '{print $2}'
-+ alarm_id=d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97
-+ sleep 1
-+ '[' -n d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 ']'
-+ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-delete d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97
-+ sleep 1
-++ openstack image list
-++ grep ' cirros '
-++ awk '{print $2}'
-+ image_id=3071d9bf-7be5-4cb1-a706-428daa446ec3
-+ sleep 1
-+ [[ true == false ]]
-+ openstack role remove admin --user doctor --project doctor
-+ openstack project delete doctor
-+ openstack user delete doctor
-+ cleanup_installer
-+ cleanup_installer_apex
-+ return
-+ cleanup_inspector
-+ cleanup_inspector_sample
-+ return
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log
deleted file mode 100755
index 0d07de02..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log
+++ /dev/null
@@ -1,9 +0,0 @@
-2016-12-28 03:15:01,868 inspector.py 50 DEBUG get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1>
- * Running on http://127.0.0.1:12345/ (Press CTRL+C to quit)
-2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3
-2016-12-28 03:16:05,299 inspector.py 77 INFO inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90>
-2016-12-28 03:16:05,299 inspector.py 78 INFO received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"}
-2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
-2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
-<Response [200]>
-127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /events HTTP/1.1" 200 -
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log
deleted file mode 100755
index a4c6ec8f..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log
+++ /dev/null
@@ -1,3 +0,0 @@
-2016-12-28 03:15:00,890 monitor.py 65 DEBUG start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5)
-2016-12-28 03:16:05,292 monitor.py 74 INFO doctor monitor detected at 1482894965.29
-2016-12-28 03:16:05,960 monitor.py 76 INFO ping timeout, quit monitoring...
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log
deleted file mode 100755
index 70cc8945..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log
+++ /dev/null
@@ -1 +0,0 @@
-Warning: Permanently added '192.0.2.6' (ECDSA) to the list of known hosts.
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log
deleted file mode 100755
index 54222d2a..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log
+++ /dev/null
@@ -1 +0,0 @@
-Warning: Permanently added '192.0.2.8' (ECDSA) to the list of known hosts.
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log
deleted file mode 100755
index 2660035b..00000000
--- a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log
+++ /dev/null
@@ -1 +0,0 @@
-Warning: Permanently added '192.0.2.9' (ECDSA) to the list of known hosts.