diff options
| -rw-r--r-- | tests/functions-common | 5 | ||||
| -rw-r--r-- | tests/lib/inspector | 2 | ||||
| -rw-r--r-- | tests/lib/inspectors/sample | 2 | ||||
| -rw-r--r-- | tests/lib/installer | 2 | ||||
| -rw-r--r-- | tests/lib/installers/apex | 2 | ||||
| -rw-r--r-- | tests/lib/installers/fuel | 197 | ||||
| -rwxr-xr-x | tests/run.sh | 6 |
7 files changed, 125 insertions, 91 deletions
diff --git a/tests/functions-common b/tests/functions-common index 08d489e5..7928f20c 100644 --- a/tests/functions-common +++ b/tests/functions-common @@ -4,7 +4,7 @@ # is_set env-var function is_set { local var=\$"$1" - eval "[ -n \"$var\" ]" + eval "[[ -n \"$var\" ]]" } # Prints backtrace info @@ -41,6 +41,8 @@ function err { # die $LINENO "message" function die { local exitcode=$? + local xtrace + xtrace=$(set +o | grep xtrace) set +o xtrace local line=$1; shift if [ $exitcode == 0 ]; then @@ -50,6 +52,7 @@ function die { err $line "$*" # Give buffers a second to flush sleep 1 + $xtrace exit $exitcode } diff --git a/tests/lib/inspector b/tests/lib/inspector index 963e9519..24221642 100644 --- a/tests/lib/inspector +++ b/tests/lib/inspector @@ -26,7 +26,7 @@ function cleanup_inspector { if ! is_inspector_supported $INSPECTOR_TYPE; then - die $LINENO"INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported." + die $LINENO "INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported." fi source $TOP_DIR/lib/inspectors/$INSPECTOR_TYPE diff --git a/tests/lib/inspectors/sample b/tests/lib/inspectors/sample index cd21a008..d09f7ad0 100644 --- a/tests/lib/inspectors/sample +++ b/tests/lib/inspectors/sample @@ -2,6 +2,8 @@ function start_inspector_sample { pgrep -f "python inspector.py" && return 0 + # TODO (r-mibu): make sample inspector use keystone v3 api + OS_AUTH_URL=${OS_AUTH_URL/v3/v2.0} \ python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 & } diff --git a/tests/lib/installer b/tests/lib/installer index 807023f3..bdee9149 100644 --- a/tests/lib/installer +++ b/tests/lib/installer @@ -29,7 +29,7 @@ function cleanup_installer { if ! is_installer_supported $INSTALLER_TYPE; then - die $LINENO"INSTALLER_TYPE=$INSTALLER_TYPE is not supported." + die $LINENO "INSTALLER_TYPE=$INSTALLER_TYPE is not supported." fi source $TOP_DIR/lib/installers/$INSTALLER_TYPE diff --git a/tests/lib/installers/apex b/tests/lib/installers/apex index 54b3dce2..45a5dcdc 100644 --- a/tests/lib/installers/apex +++ b/tests/lib/installers/apex @@ -3,7 +3,7 @@ function get_installer_ip { local instack_mac=$(sudo virsh domiflist instack | awk '/default/{print $5}') INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk '{print $1}') - die_if_not_set $LINENO $INSTALLER_IP "No installer IP" + die_if_not_set $LINENO INSTALLER_IP "No installer IP" } function installer_get_ssh_keys { diff --git a/tests/lib/installers/fuel b/tests/lib/installers/fuel index 34a86922..9ff9e724 100644 --- a/tests/lib/installers/fuel +++ b/tests/lib/installers/fuel @@ -1,107 +1,134 @@ #!/bin/bash +if [[ "congress " == "$INSPECTOR_TYPE" ]]; then + die $LINENO "fuel does not support congress yet..." +fi + function get_installer_ip { - local instack_mac=$(sudo virsh domiflist fuel-opnfv | awk '/pxebr/{print $5}') + ssh_opts_cpu="$ssh_opts -i instack_key" + is_set INSTALLER_IP && return + local instack_mac + instack_mac=$(sudo virsh domiflist fuel-master | awk '/fuel1/{print $5}') INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk '{print $1}') - die_if_not_set $LINENO $INSTALLER_IP "No installer IP" + die_if_not_set $LINENO INSTALLER_IP "No installer IP" +} + +function get_controller_ips { + is_set CONTROLLER_IPS && return + get_installer_ip + CONTROLLER_IPS=$(ssh $ssh_opts_cpu root@$INSTALLER_IP \ + "fuel node | grep controller | cut -d '|' -f 5|xargs") + die_if_not_set $LINENO CONTROLLER_IPS "No controller IPs" } function installer_get_ssh_keys { + if [[ -e instack_key ]]; then + echo "test existing instack_key..." + ssh $ssh_opts_cpu root@${INSTALLER_IP} "hostname" && return + fi + echo "getting instack_key from fuel node..." sshpass -p r00tme scp $ssh_opts root@${INSTALLER_IP}:.ssh/id_rsa instack_key sudo chown $(whoami):$(whoami) instack_key chmod 400 instack_key - ssh_opts_cpu+=" -i instack_key" } function installer_apply_patches { - cat > set_conf.sh << 'END_TXT' -#!/bin/bash -if [ -e /etc/ceilometer/event_pipeline.yaml ]; then - if ! grep -q '^ *- notifier://?topic=alarm.all$' /etc/ceilometer/event_pipeline.yaml; then - sed -i 's|- notifier://|- notifier://?topic=alarm.all|' /etc/ceilometer/event_pipeline.yaml - echo "modify the ceilometer config" - service ceilometer-agent-notification restart + if ! openstack flavor show $VM_FLAVOR ; then + openstack flavor create --ram 512 --disk 1 $VM_FLAVOR \ + && touch created_doctor_flavor fi -else - echo "ceilometer event_pipeline.yaml file does not exist" - exit 1 -fi -if [ -e /etc/nova/nova.conf ]; then - if ! grep -q '^notification_driver=messaging$' /etc/nova/nova.conf; then - sed -i -r 's/notification_driver=/notification_driver=messaging/g' /etc/nova/nova.conf - echo "modify nova config" - service nova-api restart - fi -else - echo "nova.conf file does not exist" - exit 1 -fi -exit 0 -END_TXT - - chmod +x set_conf.sh - CONTROLLER_IP=$(sshpass -p r00tme ssh 2>/dev/null $ssh_opts root@${INSTALLER_IP} \ - "fuel node | grep controller | cut -d '|' -f 5|xargs") - for node in $CONTROLLER_IP;do - scp $ssh_opts_cpu set_conf.sh "root@$node:" - ssh $ssh_opts_cpu "root@$node" './set_conf.sh > set_conf.log 2>&1 &' - sleep 1 - scp $ssh_opts_cpu "root@$node:set_conf.log" set_conf_$node.log - done - if grep -q "modify the ceilometer config" set_conf_*.log ; then - NEED_TO_RESTORE_CEILOMETER=true - fi - if grep -q "modify nova config" set_conf_*.log ; then - NEED_TO_RESTORE_NOVA=true - fi + # TODO(r-mibu): fix the followings in upstream (fuel) + get_controller_ips + for node in $CONTROLLER_IPS;do + echo "check controller configuration for doctor ($node)" + ssh $ssh_opts_cpu "root@$node" ' + set -x + date + echo "### apply patches (installer=fuel)" + + ep_conf=/etc/ceilometer/event_pipeline.yaml + entry="- notifier://?topic=alarm.all" + if ! grep -q -e "$entry" $ep_conf; then + echo "modify the ceilometer config" + echo " $entry # added by doctor script" >> $ep_conf + service ceilometer-agent-notification restart + fi - echo "waiting service restart..." - sleep 60 + # TODO(r-mibu): enable this section once congress 4.0.0 is available + if false; then + co_conf=/etc/congress/congress.conf + entry="congress.datasources.doctor_driver.DoctorDriver" + if ! grep -q -e "^drivers.*$entry" $co_conf; then + echo "modify the congress config" + sed -i -e "/^drivers/s/$/,$entry # added by doctor script/" \ + $co_conf + service congress-server restart + fi + rule="-m multiport -p tcp --dports 1789" + rule+=" -m comment --comment doctor-congress" + rule+=" -j ACCEPT" + if ! iptables -C INPUT $rule; then + iptables -I INPUT $rule + fi + + ha_conf=/etc/haproxy/conf.d/180-congress.cfg + if [[ ! -e $ha_conf ]]; then + sed -e "1i# generated by doctor script" \ + -e "s/9696/1789/" \ + -e "s/neutron/congress/" \ + /etc/haproxy/conf.d/085-neutron.cfg > $ha_conf + ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart + fi + fi + ' > installer_apply_patches_$node.log 2>&1 + done } function cleanup_installer_fuel { - if ! ($NEED_TO_RESTORE_CEILOMETER || $NEED_TO_RESTORE_NOVA) ; then - echo "Don't need to restore config" - exit 0 - fi - - echo "restore the configuration..." - cat > restore_conf.sh << 'END_TXT' -#!/bin/bash -if @NEED_TO_RESTORE_CEILOMETER@ ; then - if [ -e /etc/ceilometer/event_pipeline.yaml ]; then - if grep -q '^ *- notifier://?topic=alarm.all$' /etc/ceilometer/event_pipeline.yaml; then - sed -i 's|- notifier://?topic=alarm.all|- notifier://|' /etc/ceilometer/event_pipeline.yaml - service ceilometer-agent-notification restart - fi - else - echo "ceilometer event_pipeline.yaml file does not exist" - exit 1 + if [[ -e created_doctor_flavor ]] && openstack flavor show $VM_FLAVOR ; then + openstack flavor delete $VM_FLAVOR && rm -f created_doctor_flavor fi -fi -if @NEED_TO_RESTORE_NOVA@ ; then - if [ -e /etc/nova/nova.conf ]; then - if grep -q '^notification_driver=messaging$' /etc/nova/nova.conf; then - sed -i -r 's/notification_driver=messaging/notification_driver=/g' /etc/nova/nova.conf - service nova-api restart - fi - else - echo "nova.conf file does not exist" - exit 1 - fi -fi -exit 0 -END_TXT - sed -i -e "s/@NEED_TO_RESTORE_CEILOMETER@/$NEED_TO_RESTORE_CEILOMETER/" restore_conf.sh - sed -i -e "s/@NEED_TO_RESTORE_NOVA@/$NEED_TO_RESTORE_NOVA/" restore_conf.sh - chmod +x restore_conf.sh - for node in $CONTROLLER_IP;do - scp $ssh_opts_cpu restore_conf.sh "root@$node:" - ssh $ssh_opts_cpu "root@$node" './restore_conf.sh > restore_conf.log 2>&1 &' - done - - echo "waiting service restart..." - sleep 60 + + # TODO(r-mibu): fix the followings in upstream (fuel) + get_controller_ips + for node in $CONTROLLER_IPS;do + echo "restore controller configuration if touched ($node)" + ssh $ssh_opts_cpu "root@$node" ' + set -x + echo "### revert patches (installer=fuel)" + date + + # TODO(r-mibu): enable this section once congress 4.0.0 is available + if false; then + ha_conf=/etc/haproxy/conf.d/180-congress.cfg + if grep -q "# generated by doctor script" $ha_conf; then + rm -f $ha_conf + ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart + fi + + rule="-m multiport -p tcp --dports 1789" + rule+=" -m comment --comment doctor-congress" + rule+=" -j ACCEPT" + if iptables -C INPUT $rule; then + iptables -D INPUT $rule + fi + + co_conf=/etc/congress/congress.conf + entry="congress.datasources.doctor_driver.DoctorDriver" + if grep -q -e "^drivers.*$entry # added by doctor script" $co_conf; then + echo "modify the congress config" + sed -i -e "/^drivers/s/^\(.*\),$entry/\1/" $co_conf + service congress-server restart + fi + fi + + ep_conf=/etc/ceilometer/event_pipeline.yaml + if grep -q "# added by doctor script" $ep_conf; then + sed -ie "/# added by doctor script/d" $ep_conf + service ceilometer-agent-notification restart + fi + ' >> installer_apply_patches_$node.log 2>&1 + done } diff --git a/tests/run.sh b/tests/run.sh index a26fafee..70086502 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -179,8 +179,7 @@ start_consumer() { nova list | grep ' overcloud-controller-[0-9] ' \ | sed -e 's/^.*ctlplane=//' -e 's/ *|\$//'") elif is_installer fuel; then - CONTROLLER_IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_opts root@${INSTALLER_IP} \ - "fuel node | grep controller | cut -d '|' -f 5|xargs") + get_controller_ips fi die_if_not_set $LINENO CONTROLLER_IPS "Could not get CONTROLLER_IPS." @@ -236,6 +235,7 @@ inject_failure() { cat > disable_network.sh << 'END_TXT' #!/bin/bash -x dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $7}') +[[ -n "$dev" ]] || dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $5}') sleep 1 sudo ip link set $dev down echo "doctor set link down at" $(date "+%s.%N") @@ -289,6 +289,8 @@ unset_forced_down_hosts() { for host in $(openstack compute service list --service nova-compute \ -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p') do + # TODO (r-mibu): make sample inspector use keystone v3 api + OS_AUTH_URL=${OS_AUTH_URL/v3/v2.0} \ python ./nova_force_down.py $host --unset done |