diff options
author | dongwenjuan <dong.wenjuan@zte.com.cn> | 2018-01-10 09:42:05 +0800 |
---|---|---|
committer | dongwenjuan <dong.wenjuan@zte.com.cn> | 2018-01-10 09:42:05 +0800 |
commit | f106c5a12724c684d747cf7f309f0ccc8b86cc61 (patch) | |
tree | 43ec5c5aebd19c9ce0838033f5821d89c93f4194 /tests | |
parent | 9fd701278bf7d8c5b9451491bcb1a92580adc780 (diff) |
remove useless bash code
Change-Id: I530ef7fcdc4f9539517c5a2718e596f77de772a6
Signed-off-by: dongwenjuan <dong.wenjuan@zte.com.cn>
Diffstat (limited to 'tests')
-rw-r--r-- | tests/__init__.py | 0 | ||||
-rw-r--r-- | tests/clean.py | 45 | ||||
-rw-r--r-- | tests/consumer.py | 44 | ||||
-rw-r--r-- | tests/functions-common | 120 | ||||
-rw-r--r-- | tests/inspector.py | 122 | ||||
-rw-r--r-- | tests/lib/inspector | 32 | ||||
-rw-r--r-- | tests/lib/inspectors/congress | 69 | ||||
-rw-r--r-- | tests/lib/inspectors/sample | 18 | ||||
-rw-r--r-- | tests/lib/installer | 39 | ||||
-rw-r--r-- | tests/lib/installers/apex | 114 | ||||
-rw-r--r-- | tests/lib/installers/fuel | 197 | ||||
-rw-r--r-- | tests/lib/installers/local | 87 | ||||
-rw-r--r-- | tests/lib/monitor | 31 | ||||
-rw-r--r-- | tests/lib/monitors/collectd/collectd | 101 | ||||
-rw-r--r-- | tests/lib/monitors/collectd/collectd_plugin.py | 167 | ||||
-rw-r--r-- | tests/lib/monitors/sample/monitor.py | 124 | ||||
-rw-r--r-- | tests/lib/monitors/sample/sample | 18 | ||||
-rwxr-xr-x | tests/run.sh | 551 |
18 files changed, 0 insertions, 1879 deletions
diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/tests/__init__.py +++ /dev/null diff --git a/tests/clean.py b/tests/clean.py deleted file mode 100644 index f8394055..00000000 --- a/tests/clean.py +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2016 NEC Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import argparse -import json -import os - -import novaclient.client as novaclient - - -nova_api_version = '2.11' - -def enable_compute_host(hostname): - self.nova = novaclient.Client(self.nova_api_version, - os.environ['OS_USERNAME'], - os.environ['OS_PASSWORD'], - os.environ['OS_TENANT_NAME'], - os.environ['OS_AUTH_URL'], - connection_pool=True) - opts = {'all_tenants': True, 'host': hostname} - for server in self.nova.servers.list(detailed=False, search_opts=opts): - self.nova.servers.reset_state(server, 'active') - self.nova.services.force_down(hostname, 'nova-compute', False) - - -def get_args(): - parser = argparse.ArgumentParser(description='Doctor Test Cleaner') - parser.add_argument('hostname', metavar='HOSTNAME', type=str, nargs='?', - help='a hostname to be re-enable') - return parser.parse_args() - - -def main(): - args = get_args() - enable_compute_host(args.hostname) - - -if __name__ == '__main__': - main() diff --git a/tests/consumer.py b/tests/consumer.py deleted file mode 100644 index 042cf20a..00000000 --- a/tests/consumer.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2016 NEC Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import argparse -from flask import Flask -from flask import request -import json -import logger as doctor_log -import time - -LOG = doctor_log.Logger('doctor_consumer').getLogger() - - -app = Flask(__name__) - - -@app.route('/failure', methods=['POST']) -def event_posted(): - LOG.info('doctor consumer notified at %s' % time.time()) - LOG.info('received data = %s' % request.data) - d = json.loads(request.data) - return "OK" - - -def get_args(): - parser = argparse.ArgumentParser(description='Doctor Sample Consumer') - parser.add_argument('port', metavar='PORT', type=int, nargs='?', - help='the port for consumer') - return parser.parse_args() - - -def main(): - args = get_args() - app.run(host="0.0.0.0", port=args.port) - - -if __name__ == '__main__': - main() diff --git a/tests/functions-common b/tests/functions-common deleted file mode 100644 index 53a620b0..00000000 --- a/tests/functions-common +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash - -# Test if the named environment variable is set and not zero length -# is_set env-var -function is_set { - local var=\$"$1" - eval "[[ -n \"$var\" ]]" -} - -# Prints backtrace info -# filename:lineno:function -# backtrace level -function backtrace { - local level=$1 - local deep - deep=$((${#BASH_SOURCE[@]} - 1)) - echo "[Call Trace]" - while [ $level -le $deep ]; do - echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}" - deep=$((deep - 1)) - done -} - -# Prints line number and "message" in error format -# err $LINENO "message" -function err { - local exitcode=$? - local xtrace - xtrace=$(set +o | grep xtrace) - set +o xtrace - local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2" - echo $msg 1>&2; - if [[ -n ${LOGDIR} ]]; then - echo $msg >> "${LOGDIR}/error.log" - fi - $xtrace - return $exitcode -} - -# Prints line number and "message" then exits -# die $LINENO "message" -function die { - local exitcode=$? - local xtrace - xtrace=$(set +o | grep xtrace) - set +o xtrace - local line=$1; shift - if [ $exitcode == 0 ]; then - exitcode=1 - fi - backtrace 2 - err $line "$*" - # Give buffers a second to flush - sleep 1 - $xtrace - exit $exitcode -} - -# Checks an environment variable is not set or has length 0 OR if the -# exit code is non-zero and prints "message" and exits -# NOTE: env-var is the variable name without a '$' -# die_if_not_set $LINENO env-var "message" -function die_if_not_set { - local exitcode=$? - local xtrace - xtrace=$(set +o | grep xtrace) - set +o xtrace - local line=$1; shift - local evar=$1; shift - if ! is_set $evar || [ $exitcode != 0 ]; then - die $line "$*" - fi - $xtrace -} - -# Check the function is defined -# die_if_not_defined $LINENO function-name "message" -function die_if_not_defined { - local xtrace - xtrace=$(set +o | grep xtrace) - set +o xtrace - local line=$1; shift - local func_name=$1; shift - if ! declare -f "$func_name" > /dev/null; then - die $line "$*" - fi - $xtrace -} - -# Wait until the condition is met. -# wait_until condition timeout interval -function wait_until { - local condition="$1" - local timeout=$2 - local interval=$3 - - while eval ${condition} - do - sleep ${interval} - timeout=$((${timeout} - ${interval})) - if [[ ${timeout} < 0 ]]; then - err $LINENO "timed out ($condition)..." - return 1 - fi - done -} - -# Print IP address of the first vNIC owned by specified VM via virsh -# get_first_vnic_ip vm_name -function get_first_vnic_ip { - local vm_name=$1 - - _vnic_mac=$(sudo virsh domiflist $vm_name | \ - sed -n -e 's/^.*\([0-9a-f]\{2\}\(:[0-9a-f]\{2\}\)\{5\}\).*$/\1/p' | \ - head -1) - die_if_not_set $LINENO _vnic_mac - _vnic_ip=$(arp -e | grep $_vnic_mac | awk '{print $1}') - die_if_not_set $LINENO _vnic_ip - echo $_vnic_ip -} diff --git a/tests/inspector.py b/tests/inspector.py deleted file mode 100644 index 0046b999..00000000 --- a/tests/inspector.py +++ /dev/null @@ -1,122 +0,0 @@ -############################################################################## -# Copyright (c) 2016 NEC Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import argparse -import collections -from flask import Flask -from flask import request -import json -import logger as doctor_log -import threading -import time - -from keystoneauth1 import session -import novaclient.client as novaclient - -import doctor_tests.identity_auth - -LOG = doctor_log.Logger('doctor_inspector').getLogger() - - -class ThreadedResetState(threading.Thread): - - def __init__(self, nova, state, server): - threading.Thread.__init__(self) - self.nova = nova - self.state = state - self.server = server - - def run(self): - self.nova.servers.reset_state(self.server, self.state) - LOG.info('doctor mark vm(%s) error at %s' % (self.server, time.time())) - - -class DoctorInspectorSample(object): - - NOVA_API_VERSION = '2.34' - NUMBER_OF_CLIENTS = 50 - # TODO(tojuvone): This could be enhanced in future with dynamic - # reuse of self.novaclients when all threads in use and - # self.NUMBER_OF_CLIENTS based on amount of cores or overriden by input - # argument - - def __init__(self): - self.servers = collections.defaultdict(list) - self.novaclients = list() - auth=identity_auth.get_identity_auth() - sess=session.Session(auth=auth) - # Pool of novaclients for redundant usage - for i in range(self.NUMBER_OF_CLIENTS): - self.novaclients.append( - novaclient.Client(self.NOVA_API_VERSION, session=sess)) - # Normally we use this client for non redundant API calls - self.nova=self.novaclients[0] - self.nova.servers.list(detailed=False) - self.init_servers_list() - - def init_servers_list(self): - opts = {'all_tenants': True} - servers=self.nova.servers.list(search_opts=opts) - self.servers.clear() - for server in servers: - try: - host=server.__dict__.get('OS-EXT-SRV-ATTR:host') - self.servers[host].append(server) - LOG.debug('get hostname=%s from server=%s' % (host, server)) - except Exception as e: - LOG.error('can not get hostname from server=%s' % server) - - def disable_compute_host(self, hostname): - threads = [] - if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS: - # TODO(tojuvone): This could be enhanced in future with dynamic - # reuse of self.novaclients when all threads in use - LOG.error('%d servers in %s. Can handle only %d'%( - self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS)) - for nova, server in zip(self.novaclients, self.servers[hostname]): - t = ThreadedResetState(nova, "error", server) - t.start() - threads.append(t) - for t in threads: - t.join() - self.nova.services.force_down(hostname, 'nova-compute', True) - LOG.info('doctor mark host(%s) down at %s' % (hostname, time.time())) - - -app = Flask(__name__) -inspector = DoctorInspectorSample() - - -@app.route('/events', methods=['POST']) -def event_posted(): - LOG.info('event posted at %s' % time.time()) - LOG.info('inspector = %s' % inspector) - LOG.info('received data = %s' % request.data) - d = json.loads(request.data) - for event in d: - hostname = event['details']['hostname'] - event_type = event['type'] - if event_type == 'compute.host.down': - inspector.disable_compute_host(hostname) - return "OK" - - -def get_args(): - parser = argparse.ArgumentParser(description='Doctor Sample Inspector') - parser.add_argument('port', metavar='PORT', type=int, nargs='?', - help='a port for inspector') - return parser.parse_args() - - -def main(): - args = get_args() - app.run(host='0.0.0.0', port=args.port) - -if __name__ == '__main__': - main() diff --git a/tests/lib/inspector b/tests/lib/inspector deleted file mode 100644 index 24221642..00000000 --- a/tests/lib/inspector +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -INSPECTOR_TYPE=${INSPECTOR_TYPE:-sample} - -function is_inspector_supported { - local inspector="$1" - [[ -f $TOP_DIR/lib/inspectors/$inspector ]] -} - -function is_inspector { - local inspector="$1" - [[ $inspector == $INSPECTOR_TYPE ]] -} - -function start_inspector { - start_inspector_$INSPECTOR_TYPE -} - -function stop_inspector { - stop_inspector_$INSPECTOR_TYPE -} - -function cleanup_inspector { - cleanup_inspector_$INSPECTOR_TYPE -} - - -if ! is_inspector_supported $INSPECTOR_TYPE; then - die $LINENO "INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported." -fi - -source $TOP_DIR/lib/inspectors/$INSPECTOR_TYPE diff --git a/tests/lib/inspectors/congress b/tests/lib/inspectors/congress deleted file mode 100644 index 04825252..00000000 --- a/tests/lib/inspectors/congress +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -function _congress_add_rule { - name=$1 - policy=$2 - rule=$3 - - if ! openstack congress policy rule list $policy | grep -q -e "// Name: $name$" ; then - openstack congress policy rule create --name $name $policy "$rule" - fi -} - -function _congress_del_rule { - name=$1 - policy=$2 - - if openstack congress policy rule list $policy | grep -q -e "^// Name: $name$" ; then - openstack congress policy rule delete $policy $name - fi -} - -function _congress_add_rules { - _congress_add_rule host_down classification \ - 'host_down(host) :- - doctor:events(hostname=host, type="compute.host.down", status="down")' - - _congress_add_rule active_instance_in_host classification \ - 'active_instance_in_host(vmid, host) :- - nova:servers(id=vmid, host_name=host, status="ACTIVE")' - - _congress_add_rule host_force_down classification \ - 'execute[nova:services.force_down(host, "nova-compute", "True")] :- - host_down(host)' - - _congress_add_rule error_vm_states classification \ - 'execute[nova:servers.reset_state(vmid, "error")] :- - host_down(host), - active_instance_in_host(vmid, host)' -} - -function start_inspector_congress { - nova_api_min_version="2.11" - nova_api_version=$(openstack congress datasource list | \ - grep nova | grep -Po "(?<='api_version': ')[^']*") - [[ -z $nova_api_version ]] && nova_api_version="2.0" - if [[ "$nova_api_version" < "$nova_api_min_version" ]]; then - echo "ERROR: Congress Nova datasource API version < $nova_api_min_version ($nova_api_version)" - exit 1 - fi - openstack congress driver list | grep -q " doctor " - openstack congress datasource list | grep -q " doctor " || { - openstack congress datasource create doctor doctor - } - _congress_add_rules - -} - -function stop_inspector_congress { - _congress_del_rule host_force_down classification - _congress_del_rule error_vm_states classification - _congress_del_rule active_instance_in_host classification - _congress_del_rule host_down classification - -} - -function cleanup_inspector_congress { - # Noop - return -} diff --git a/tests/lib/inspectors/sample b/tests/lib/inspectors/sample deleted file mode 100644 index 75d5c022..00000000 --- a/tests/lib/inspectors/sample +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -INSPECTOR_PORT=${INSPECTOR_PORT:-12345} - -function start_inspector_sample { - pgrep -f "python inspector.py" && return 0 - python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 & -} - -function stop_inspector_sample { - pgrep -f "python inspector.py" || return 0 - kill $(pgrep -f "python inspector.py") -} - -function cleanup_inspector_sample { - # Noop - return -} diff --git a/tests/lib/installer b/tests/lib/installer deleted file mode 100644 index 13953d28..00000000 --- a/tests/lib/installer +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -INSTALLER_TYPE=${INSTALLER_TYPE:-local} -ssh_opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - -function is_installer_supported { - local installer="$1" - [[ -f $TOP_DIR/lib/installers/$installer ]] -} - -function is_installer { - local installer="$1" - [[ $installer == $INSTALLER_TYPE ]] -} - -function validate_installer_lib { - local xtrace - xtrace=$(set +o | grep xtrace) - set +o xtrace - for p in COMPUTE_USER ssh_opts_cpu - do - die_if_not_set $LINENO $p \ - "Parameter $p for $INSTALLER_TYPE is missing." - done - for f in setup_installer get_compute_ip_from_hostname cleanup_installer - do - die_if_not_defined $LINENO $f \ - "Mandatory function ${f}() for $INSTALLER_TYPE is missing." - done - $xtrace -} - -if ! is_installer_supported $INSTALLER_TYPE; then - die $LINENO "INSTALLER_TYPE=$INSTALLER_TYPE is not supported." -fi - -source $TOP_DIR/lib/installers/$INSTALLER_TYPE - -validate_installer_lib diff --git a/tests/lib/installers/apex b/tests/lib/installers/apex deleted file mode 100644 index f7b9624e..00000000 --- a/tests/lib/installers/apex +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -COMPUTE_USER=${COMPUTE_USER:-heat-admin} -ssh_opts_cpu="$ssh_opts -i instack_key" - -function get_installer_ip { - is_set INSTALLER_IP && return - INSTALLER_IP=$(get_first_vnic_ip undercloud) -} - -function installer_get_ssh_keys { - sudo scp $ssh_opts "root@$INSTALLER_IP:/home/stack/.ssh/id_rsa" instack_key - sudo chown $(whoami):$(whoami) instack_key - chmod 400 instack_key -} - -function get_controller_ips { - is_set CONTROLLER_IPS && return - get_installer_ip - CONTROLLER_IPS=$(sudo ssh $ssh_opts $INSTALLER_IP \ - "source stackrc - nova list | grep ' overcloud-controller-[0-9] ' | \ - sed -e 's/^.*ctlplane=//' -e 's/ *|\$//'") - die_if_not_set $LINENO CONTROLLER_IPS "No controller IPs" -} - -function installer_apply_patches { - # TODO(r-mibu): fix the followings in upstream (apex) - for node in $CONTROLLER_IPS;do - echo "check controller configuration for doctor ($node)" - ssh $ssh_opts_cpu "heat-admin@$node" ' - set -x - date - echo "### apply patches (installer=apex)" - - ep_conf=/etc/ceilometer/event_pipeline.yaml - ep_entry="- notifier://?topic=alarm.all" - if sudo grep -e "$ep_entry" $ep_conf; then - echo "NOTE: ceilometer is configured as we needed" - else - echo "modify the ceilometer config" - sudo sed -i -e "$ a \ \ \ \ \ \ \ \ \ \ $ep_entry # added by doctor script" $ep_conf - sudo systemctl restart openstack-ceilometer-notification.service - fi - - co_conf=/etc/congress/congress.conf - co_conf_bak=/etc/congress/congress.conf.bak - co_entry="congress.datasources.doctor_driver.DoctorDriver" - if sudo grep -e "^drivers.*$co_entry" $co_conf; then - echo "NOTE: congress is configured as we needed" - else - echo "modify the congress config" - sudo cp $co_conf $co_conf_bak - sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf - sudo systemctl restart openstack-congress-server.service - fi - ' > installer_apply_patches_$node.log 2>&1 - done -} - -function setup_installer { - get_installer_ip - installer_get_ssh_keys - get_controller_ips - installer_apply_patches - - # NOTE: while executing command as doctor user, - # 'OS_PROJECT_ID' env parameter make openstack clients confused. - unset OS_PROJECT_ID -} - -function get_compute_ip_from_hostname { - local compute_host=$1 - - compute_host_in_undercloud=${compute_host%%.*} - COMPUTE_IP=$(sudo ssh $ssh_opts $INSTALLER_IP \ - "source stackrc; - nova show $compute_host_in_undercloud | \ - awk '/ ctlplane network /{print \$5}'") - die_if_not_set $LINENO COMPUTE_IP "Could get IP address of $compute_host." -} - -function installer_revert_patches { - # TODO(r-mibu): fix the followings in upstream (apex) - get_controller_ips - for node in $CONTROLLER_IPS;do - echo "restore controller configuration if touched ($node)" - ssh $ssh_opts_cpu "heat-admin@$node" ' - set -x - echo "### revert patches (installer=apex)" - date - - co_conf=/etc/congress/congress.conf - co_conf_bak=/etc/congress/congress.conf.bak - if [ -e $co_conf_bak ]; then - echo "restore the congress config" - sudo mv $co_conf_bak $co_conf - sudo systemctl restart openstack-congress-server.service - fi - - ep_conf=/etc/ceilometer/event_pipeline.yaml - if sudo grep -q -e "# added by doctor script" $ep_conf; then - echo "modify the ceilometer config" - sudo sed -ie "/# added by doctor script/d" $ep_conf - sudo systemctl restart openstack-ceilometer-notification.service - fi - ' >> installer_apply_patches_$node.log 2>&1 - done -} - -function cleanup_installer { - installer_revert_patches - return -} diff --git a/tests/lib/installers/fuel b/tests/lib/installers/fuel deleted file mode 100644 index 85865720..00000000 --- a/tests/lib/installers/fuel +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash - -if [[ "congress " == "$INSPECTOR_TYPE" ]]; then - die $LINENO "fuel does not support congress yet..." -fi - -COMPUTE_USER=${COMPUTE_USER:-root} -ssh_opts_cpu="$ssh_opts -i instack_key" - -function get_installer_ip { - is_set INSTALLER_IP && return - INSTALLER_IP=$(get_first_vnic_ip fuel-master) -} - -function get_controller_ips { - is_set CONTROLLER_IPS && return - CONTROLLER_IPS=$(ssh $ssh_opts_cpu root@$INSTALLER_IP \ - "fuel node | grep controller | cut -d '|' -f 5|xargs") - die_if_not_set $LINENO CONTROLLER_IPS "No controller IPs" -} - -function installer_get_ssh_keys { - if [[ -e instack_key ]]; then - echo "test existing instack_key..." - ssh $ssh_opts_cpu root@${INSTALLER_IP} "hostname" && return - fi - echo "getting instack_key from fuel node..." - sshpass -p r00tme scp $ssh_opts root@${INSTALLER_IP}:.ssh/id_rsa instack_key - sudo chown $(whoami):$(whoami) instack_key - chmod 400 instack_key -} - -function installer_apply_patches { - # TODO(r-mibu): fix the followings in upstream (fuel) - for node in $CONTROLLER_IPS;do - echo "check controller configuration for doctor ($node)" - ssh $ssh_opts_cpu "root@$node" ' - set -x - date - echo "### apply patches (installer=fuel)" - - ep_conf=/etc/ceilometer/event_pipeline.yaml - entry="- notifier://?topic=alarm.all" - if ! grep -q -e "$entry" $ep_conf; then - echo "modify the ceilometer config" - echo " $entry # added by doctor script" >> $ep_conf - service ceilometer-agent-notification restart - fi - - # TODO(r-mibu): enable this section once congress 4.0.0 is available - if false; then - co_conf=/etc/congress/congress.conf - entry="congress.datasources.doctor_driver.DoctorDriver" - if ! grep -q -e "^drivers.*$entry" $co_conf; then - echo "modify the congress config" - sed -i -e "/^drivers/s/$/,$entry # added by doctor script/" \ - $co_conf - service congress-server restart - fi - - rule="-m multiport -p tcp --dports 1789" - rule+=" -m comment --comment doctor-congress" - rule+=" -j ACCEPT" - if ! iptables -C INPUT $rule; then - iptables -I INPUT $rule - fi - - ha_conf=/etc/haproxy/conf.d/180-congress.cfg - if [[ ! -e $ha_conf ]]; then - sed -e "1i# generated by doctor script" \ - -e "s/9696/1789/" \ - -e "s/neutron/congress/" \ - /etc/haproxy/conf.d/085-neutron.cfg > $ha_conf - ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart - fi - fi - - np_conf=/etc/nova/policy.json - if [ -e $np_conf ]; then - entry="os_compute_api:servers:show:host_status" - new="rule:admin_or_owner" - np_backup="${np_conf}-doctor-saved" - if grep -q "${entry}.*${new}" $np_conf; then - echo "Not modifying nova policy" - elif grep -q "${entry}" $np_conf; then - echo "modify nova policy" - cp $np_conf $np_backup - oldline=$(grep "$entry" $np_conf) - newline=$(echo "$oldline" | sed "s/rule.*\"/$new\"/") - sed -i "s/$oldline/$newline/" $np_conf - service nova-api restart - else - echo "add nova policy" - cp $np_conf $np_backup - sed -i "/{/a \ \"${entry}\": \"$new\"" $np_conf - service nova-api restart - fi - else - # policy.json does not exist in Ocata. - echo "$np_conf does not exist. Creating new one." - echo -e "{\n \"context_is_admin\": \"role:admin\"," > $np_conf - echo -e " \"owner\" : \"user_id:%(user_id)s\"," >> $np_conf - echo -e " \"admin_or_owner\": \"rule:context_is_admin or rule:owner\"," >> $np_conf - echo -e " \"os_compute_api:servers:show:host_status\": \"rule:admin_or_owner\" \n}" >> $np_conf - np_rm="${np_conf}-doctor-rm" - cp $np_conf $np_rm - service nova-api restart - fi - ' > installer_apply_patches_$node.log 2>&1 - done -} - -function setup_installer { - get_installer_ip - installer_get_ssh_keys - get_controller_ips - installer_apply_patches - #Might take a moment for nova-api to restart - sleep 20 - if ! openstack flavor show $VM_FLAVOR ; then - openstack flavor create --ram 512 --disk 1 $VM_FLAVOR \ - && touch created_doctor_flavor - fi -} - -function get_compute_ip_from_hostname { - local compute_host=$1 - - compute_host_in_undercloud=${compute_host%%.*} - node_id=$(echo $compute_host_in_undercloud | cut -d "-" -f 2) - COMPUTE_IP=$(sshpass -p r00tme ssh 2>/dev/null $ssh_opts root@${INSTALLER_IP} \ - "fuel node|awk -F '|' -v id=$node_id '{if (\$1 == id) print \$5}' |xargs") - die_if_not_set $LINENO COMPUTE_IP "Could get IP address of $compute_host." -} - -function installer_revert_patches { - # TODO(r-mibu): fix the followings in upstream (fuel) - get_controller_ips - for node in $CONTROLLER_IPS;do - echo "restore controller configuration if touched ($node)" - ssh $ssh_opts_cpu "root@$node" ' - set -x - echo "### revert patches (installer=fuel)" - date - - # TODO(r-mibu): enable this section once congress 4.0.0 is available - if false; then - ha_conf=/etc/haproxy/conf.d/180-congress.cfg - if grep -q "# generated by doctor script" $ha_conf; then - rm -f $ha_conf - ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart - fi - - rule="-m multiport -p tcp --dports 1789" - rule+=" -m comment --comment doctor-congress" - rule+=" -j ACCEPT" - if iptables -C INPUT $rule; then - iptables -D INPUT $rule - fi - - co_conf=/etc/congress/congress.conf - entry="congress.datasources.doctor_driver.DoctorDriver" - if grep -q -e "^drivers.*$entry # added by doctor script" $co_conf; then - echo "modify the congress config" - sed -i -e "/^drivers/s/^\(.*\),$entry/\1/" $co_conf - service congress-server restart - fi - fi - - ep_conf=/etc/ceilometer/event_pipeline.yaml - if grep -q "# added by doctor script" $ep_conf; then - sed -ie "/# added by doctor script/d" $ep_conf - service ceilometer-agent-notification restart - fi - - np_conf=/etc/nova/policy.json - np_backup="${np_conf}-doctor-saved" - np_rm="${np_conf}-doctor-rm" - if [ -e $np_backup ]; then - cp -f $np_backup $np_conf - rm $np_backup - service nova-api restart - elif [ -e $np_rm ]; then - rm $np_conf - rm $np_rm - service nova-api restart - fi - ' >> installer_apply_patches_$node.log 2>&1 - done -} - -function cleanup_installer { - if [[ -e created_doctor_flavor ]] && openstack flavor show $VM_FLAVOR ; then - openstack flavor delete $VM_FLAVOR && rm -f created_doctor_flavor - fi - installer_revert_patches -} diff --git a/tests/lib/installers/local b/tests/lib/installers/local deleted file mode 100644 index d628867a..00000000 --- a/tests/lib/installers/local +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -COMPUTE_USER=${COMPUTE_USER:-$(whoami)} -ssh_opts_cpu="$ssh_opts" - -function installer_get_ssh_keys { - echo "INSTALLER_TYPE set to 'local'. Assuming SSH keys already exchanged with $COMPUTE_HOST" - return -} - -function installer_apply_patches { - set -x - date - echo "### apply patches (installer=local)" - np_conf=/etc/nova/policy.json - if [ -e $np_conf ]; then - entry="os_compute_api:servers:show:host_status" - new="rule:admin_or_owner" - np_backup="${np_conf}-doctor-saved" - if grep -q "${entry}.*${new}" $np_conf; then - echo "Not modifying nova policy" - elif grep -q "${entry}" $np_conf; then - echo "modify nova policy" - cp $np_conf $np_backup - oldline=$(grep "$entry" $np_conf) - newline=$(echo "$oldline" | sed "s/rule.*\"/$new\"/") - sed -i "s/$oldline/$newline/" $np_conf - # TODO(umar): Update to systemd when screen is no more used for devstack - screen -S stack -p n-api -X stuff "^C^M^[[A^M" # restart n-api service - else - echo "add nova policy" - cp $np_conf $np_backup - sed -i "/{/a \ \"${entry}\": \"$new\"" $np_conf - screen -S stack -p n-api -X stuff "^C^M^[[A^M" - fi - else - # policy.json does not exist in Ocata. - echo "$np_conf does not exist. Creating a new one" - echo -e '{\n "context_is_admin": "role:admin",' > $np_conf - echo -e ' "owner" : "user_id:%(user_id)s",' >> $np_conf - echo -e ' "admin_or_owner": "rule:context_is_admin or rule:owner",' >> $np_conf - echo -e ' "os_compute_api:servers:show:host_status": "rule:admin_or_owner"\n}' >> $np_conf - np_rm="${np_conf}-doctor-rm" - cp $np_conf $np_rm - screen -S stack -p n-api -X stuff "^C^M^[[A^M" - fi - - return -} - -function setup_installer { - installer_get_ssh_keys - installer_apply_patches -} - -function get_compute_ip_from_hostname { - local compute_host=$1 - - if is_set COMPUTE_IP; then - echo "Using pre-configured COMPUTE_IP=$COMPUTE_IP ..." - return - fi - COMPUTE_IP=$(getent hosts "$compute_host" | awk '{ print $1 }') - die_if_not_set $LINENO COMPUTE_IP \ - "Could not resolve $compute_host. Either manually set COMPUTE_IP or enable DNS resolution." -} - -function cleanup_installer { - set -x - echo "### revert patches (installer=local)" - date - - np_conf=/etc/nova/policy.json - np_backup="${np_conf}-doctor-saved" - np_rm="${np_conf}-doctor-rm" - if [ -e $np_backup ]; then - cp -f $np_backup $np_conf - rm $np_backup - screen -S stack -p n-api -X stuff "^C^M^[[A^M" - elif [ -e $np_rm ]; then - rm $np_conf - rm $np_rm - screen -S stack -p n-api -X stuff "^C^M^[[A^M" - fi - - return -} diff --git a/tests/lib/monitor b/tests/lib/monitor deleted file mode 100644 index 6b804ec2..00000000 --- a/tests/lib/monitor +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -MONITOR_TYPE=${MONITOR_TYPE:-sample} - -function is_monitor_supported { - local monitor="$1" - [[ -f $TOP_DIR/lib/monitors/$monitor/$monitor ]] -} - -function is_monitor { - local monitor="$1" - [[ $monitor == $MONITOR_TYPE ]] -} - -function start_monitor { - start_monitor_$MONITOR_TYPE -} - -function stop_monitor { - stop_monitor_$MONITOR_TYPE -} - -function cleanup_monitor { - cleanup_monitor_$MONITOR_TYPE -} - -if ! is_monitor_supported $MONITOR_TYPE; then - die $LINENO "MONITOR_TYPE=$MONITOR_TYPE is not supported." -fi - -source $TOP_DIR/lib/monitors/$MONITOR_TYPE/$MONITOR_TYPE diff --git a/tests/lib/monitors/collectd/collectd b/tests/lib/monitors/collectd/collectd deleted file mode 100644 index f5096658..00000000 --- a/tests/lib/monitors/collectd/collectd +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -function start_monitor_collectd { - ## CONTROL_IP is the IP of primary interface of control node i.e. - ## eth0, eno1. It is used by collectd monitor to communicate with - ## sample inspector. - ## @TODO (umar) see if mgmt IP of control is a better option. Also - ## primary interface may not be the right option - CONTROL_IP="$(ip a | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | sed -n 1p)" - #CONTROL_IP=192.168.98.6 - - echo " -Hostname \"$COMPUTE_HOST\" -FQDNLookup false -Interval 1 -MaxReadInterval 2 - -<LoadPlugin python> - Globals true -</LoadPlugin> -LoadPlugin ovs_events -LoadPlugin logfile - -<Plugin logfile> - File \"/var/log/collectd.log\" - Timestamp true - LogLevel \"info\" -</Plugin> - -<Plugin python> - ModulePath \"/home/$COMPUTE_USER\" - LogTraces true - Interactive false - Import \"collectd_plugin\" - <Module \"collectd_plugin\"> - control_ip \"$CONTROL_IP\" - compute_ip \"$COMPUTE_IP\" - compute_host \"$COMPUTE_HOST\" - compute_user \"$COMPUTE_USER\" - inspector_type \"$INSPECTOR_TYPE\" - os_auth_url \"$OS_AUTH_URL\" - os_username \"$OS_USERNAME\" - os_password \"$OS_PASSWORD\" - os_project_name \"$OS_PROJECT_NAME\" - os_user_domain_name \"$OS_USER_DOMAIN_NAME\" - os_user_domain_id \"$OS_USER_DOMAIN_ID\" - os_project_domain_name \"$OS_PROJECT_DOMAIN_NAME\" - os_project_domain_id \"$OS_PROJECT_DOMAIN_ID\" - </Module> -</Plugin> - -<Plugin ovs_events> - Port 6640 - Socket \"/var/run/openvswitch/db.sock\" - Interfaces \"@INTERFACE_NAME@\" - SendNotification true - DispatchValues false -</Plugin> - -" > $TOP_DIR/lib/monitors/collectd.conf - - scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd.conf $COMPUTE_USER@$COMPUTE_IP: - ## @TODO (umar) Always assuming that the interface is assigned an IP if - ## interface name is not provided. See if there is a better approach - ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" " - if [ -n \"$INTERFACE_NAME\" ]; then - dev=$INTERFACE_NAME - else - dev=\$(sudo ip a | awk '/ $COMPUTE_IP\//{print \$NF}') - fi - sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf - collectd_conf=/opt/collectd/etc/collectd.conf - if [ -e \$collectd_conf ]; then - sudo cp \$collectd_conf \${collectd_conf}-doctor-saved - else - sudo touch \${collectd_conf}-doctor-created - fi - sudo mv collectd.conf /opt/collectd/etc/collectd.conf" - - scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd/collectd_plugin.py $COMPUTE_USER@$COMPUTE_IP:collectd_plugin.py - ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "sudo pkill collectd - sudo /opt/collectd/sbin/collectd" -} - -function stop_monitor_collectd { - ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'sudo pkill collectd' -} - -function cleanup_monitor_collectd { - ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" " - collectd_conf=/opt/collectd/etc/collectd.conf - if [ -e \"\${collectd_conf}-doctor-created\" ]; then - sudo rm \"\${collectd_conf}-doctor-created\" - sudo rm \$collectd_conf - elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then - sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf - sudo rm \"\${collectd_conf}-doctor-saved\" - fi" - - rm $TOP_DIR/lib/monitors/collectd.conf -} diff --git a/tests/lib/monitors/collectd/collectd_plugin.py b/tests/lib/monitors/collectd/collectd_plugin.py deleted file mode 100644 index 70fcf26e..00000000 --- a/tests/lib/monitors/collectd/collectd_plugin.py +++ /dev/null @@ -1,167 +0,0 @@ -############################################################################## -# Copyright (c) 2017 NEC Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import collectd -import sys -from netifaces import interfaces, ifaddresses, AF_INET -from datetime import datetime -import json -import requests -import time -from requests.exceptions import ConnectionError - -from keystoneauth1 import loading -from keystoneauth1 import session -from congressclient.v1 import client - - -def write_debug(str_write, write_type, compute_user): - file_name = ('/home/%s/monitor.log' % compute_user) - file_tmp = open(file_name, write_type) - file_tmp.write( "%s" % str_write) - file_tmp.close() - - -class DoctorMonitorCollectd(object): - def __init__(self): - self.control_ip = '' - self.compute_user = '' - self.compute_ip = '' - self.host_name = '' - self.inspector_type = '' - self.inspector_url = '' - self.os_auth_url = '' - self.os_username = '' - self.os_password = '' - self.os_project_name = '' - self.os_user_domain_name = '' - self.os_user_domain_id = '' - self.os_project_domain_name = '' - self.os_project_domain_id = '' - self.sess = '' - self.auth = '' - self.inspector_notified = 0 - self.start_notifications = 0 - self.monitor_type = 'sample' - - def config_func(self, config): - for node in config.children: - key = node.key.lower() - val = node.values[0] - - if key == 'compute_host': - self.host_name = val - elif key == 'control_ip': - self.control_ip = val - elif key == 'compute_ip': - self.compute_ip = val - elif key == 'compute_user': - self.compute_user = val - elif key == 'inspector_type': - self.inspector_type = val - elif key == 'os_auth_url': - self.os_auth_url = val - elif key == 'os_username': - self.os_username = val - elif key == 'os_password': - self.os_password = val - elif key == 'os_project_name': - self.os_project_name = val - elif key == 'os_user_domain_name': - self.os_user_domain_name = val - elif key == 'os_user_domain_id': - self.os_user_domain_id = val - elif key == 'os_project_domain_name': - self.os_project_domain_name = val - elif key == 'os_project_domain_id': - self.os_project_domain_id = val - else: - collectd.info('Unknown config key "%s"' % key) - - def init_collectd(self): - write_debug("Compute node collectd monitor start at %s\n\n" % datetime.now().isoformat(), "w", self.compute_user) - - if self.inspector_type == 'sample': - self.inspector_url = ('http://%s:12345/events' % self.control_ip) - elif self.inspector_type == 'congress': - loader = loading.get_plugin_loader('password') - self.auth = loader.load_from_options(auth_url=self.os_auth_url, - username=self.os_username, - password=self.os_password, - project_name=self.os_project_name, - user_domain_name=self.os_user_domain_name, - user_domain_id=self.os_user_domain_id, - project_domain_name=self.os_project_domain_name, - project_domain_id=self.os_project_domain_id) - self.sess=session.Session(auth=self.auth) - congress = client.Client(session=self.sess, service_type='policy') - ds = congress.list_datasources()['results'] - doctor_ds = next((item for item in ds if item['driver'] == 'doctor'), - None) - - congress_endpoint = congress.httpclient.get_endpoint(auth=self.auth) - self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' % - (congress_endpoint, doctor_ds['id'])) - else: - sys.exit() - self.start_notifications = 1 - - - def notify_inspector(self): - event_type = "compute.host.down" - payload = [ - { - 'id': ("monitor_%s_id1" % self.monitor_type), - 'time': datetime.now().isoformat(), - 'type': event_type, - 'details': { - 'hostname': self.host_name, - 'status': 'down', - 'monitor': ("monitor_%s" % self.monitor_type), - 'monitor_event_id': ("monitor_%s_event1" % self.monitor_type) - }, - }, - ] - data = json.dumps(payload) - self.inspector_notified = 1 - - if self.inspector_type == 'sample': - headers = {'content-type': 'application/json'} - try: - requests.post(self.inspector_url, data=data, headers=headers) - except ConnectionError as err: - print err - elif self.inspector_type == 'congress': - # TODO(umar) enhance for token expiry case - headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - 'X-Auth-Token': self.sess.get_token() - } - requests.put(self.inspector_url, data=data, headers=headers) - - - def handle_notif(self, notification, data=None): - if (notification.severity == collectd.NOTIF_FAILURE or - notification.severity == collectd.NOTIF_WARNING): - if (self.start_notifications == 1 and self.inspector_notified == 0): - write_debug("Received down notification: doctor monitor detected at %s\n" % time.time(), "a", self.compute_user) - self.notify_inspector() - - elif notification.severity == collectd.NOTIF_OKAY: - collectd.info("Interface status: UP again %s\n" % time.time()) - else: - collectd.info("Unknown notification severity %s\n" % notification.severity) - - -monitor = DoctorMonitorCollectd() - -collectd.register_config(monitor.config_func) -collectd.register_init(monitor.init_collectd) -collectd.register_notification(monitor.handle_notif) diff --git a/tests/lib/monitors/sample/monitor.py b/tests/lib/monitors/sample/monitor.py deleted file mode 100644 index 7450c534..00000000 --- a/tests/lib/monitors/sample/monitor.py +++ /dev/null @@ -1,124 +0,0 @@ -############################################################################## -# Copyright (c) 2016 NEC Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import argparse -from datetime import datetime -import json -import logger as doctor_log -import requests -import socket -import time - -from keystoneauth1 import session -from congressclient.v1 import client - -import identity_auth - -# NOTE: icmp message with all zero data (checksum = 0xf7ff) -# see https://tools.ietf.org/html/rfc792 -ICMP_ECHO_MESSAGE = '\x08\x00\xf7\xff\x00\x00\x00\x00' - -SUPPORTED_INSPECTOR_TYPES = ['sample', 'congress'] - -LOG = doctor_log.Logger('doctor_monitor').getLogger() - - -class DoctorMonitorSample(object): - - interval = 0.1 # second - timeout = 0.1 # second - event_type = "compute.host.down" - - def __init__(self, args): - if args.inspector_type not in SUPPORTED_INSPECTOR_TYPES: - raise Exception("Inspector type '%s' not supported", args.inspector_type) - - self.hostname = args.hostname - self.inspector_type = args.inspector_type - self.ip_addr = args.ip or socket.gethostbyname(self.hostname) - - if self.inspector_type == 'sample': - self.inspector_url = 'http://127.0.0.1:12345/events' - elif self.inspector_type == 'congress': - auth=identity_auth.get_identity_auth() - self.session=session.Session(auth=auth) - congress = client.Client(session=self.session, service_type='policy') - ds = congress.list_datasources()['results'] - doctor_ds = next((item for item in ds if item['driver'] == 'doctor'), - None) - - congress_endpoint = congress.httpclient.get_endpoint(auth=auth) - self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' % - (congress_endpoint, doctor_ds['id'])) - - def start_loop(self): - LOG.debug("start ping to host %(h)s (ip=%(i)s)" % {'h': self.hostname, - 'i': self.ip_addr}) - sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, - socket.IPPROTO_ICMP) - sock.settimeout(self.timeout) - while True: - try: - sock.sendto(ICMP_ECHO_MESSAGE, (self.ip_addr, 0)) - data = sock.recv(4096) - except socket.timeout: - LOG.info("doctor monitor detected at %s" % time.time()) - self.report_error() - LOG.info("ping timeout, quit monitoring...") - return - time.sleep(self.interval) - - def report_error(self): - payload = [ - { - 'id': 'monitor_sample_id1', - 'time': datetime.now().isoformat(), - 'type': self.event_type, - 'details': { - 'hostname': self.hostname, - 'status': 'down', - 'monitor': 'monitor_sample', - 'monitor_event_id': 'monitor_sample_event1' - }, - }, - ] - data = json.dumps(payload) - - if self.inspector_type == 'sample': - headers = {'content-type': 'application/json'} - requests.post(self.inspector_url, data=data, headers=headers) - elif self.inspector_type == 'congress': - headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - 'X-Auth-Token':self.session.get_token(), - } - requests.put(self.inspector_url, data=data, headers=headers) - - -def get_args(): - parser = argparse.ArgumentParser(description='Doctor Sample Monitor') - parser.add_argument('hostname', metavar='HOSTNAME', type=str, nargs='?', - help='a hostname to monitor connectivity') - parser.add_argument('ip', metavar='IP', type=str, nargs='?', - help='an IP address to monitor connectivity') - parser.add_argument('inspector_type', metavar='INSPECTOR_TYPE', type=str, nargs='?', - help='inspector to report', - default='sample') - return parser.parse_args() - - -def main(): - args = get_args() - monitor = DoctorMonitorSample(args) - monitor.start_loop() - - -if __name__ == '__main__': - main() diff --git a/tests/lib/monitors/sample/sample b/tests/lib/monitors/sample/sample deleted file mode 100644 index 1d310333..00000000 --- a/tests/lib/monitors/sample/sample +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -function start_monitor_sample { - cp $TOP_DIR/lib/monitors/sample/monitor.py $TOP_DIR/monitor.py - pgrep -f "python monitor.py" && return 0 - sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \ - > monitor.log 2>&1 & -} - -function stop_monitor_sample { - pgrep -f "python monitor.py" || return 0 - sudo kill $(pgrep -f "python monitor.py") -} - -function cleanup_monitor_sample { - rm monitor.py - return -} diff --git a/tests/run.sh b/tests/run.sh deleted file mode 100755 index b5c56872..00000000 --- a/tests/run.sh +++ /dev/null @@ -1,551 +0,0 @@ -#!/bin/bash -e -############################################################################## -# Copyright (c) 2016 NEC Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# Configuration - -[[ "${CI_DEBUG:-true}" == [Tt]rue ]] && set -x - -IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img -#if an existing image name is provided in the enviroment, use that one -IMAGE_NAME=${IMAGE_NAME:-cirros} -IMAGE_FILE="${IMAGE_NAME}.img" -IMAGE_FORMAT=qcow2 -VM_BASENAME=doctor_vm -VM_FLAVOR=m1.tiny -#if VM_COUNT set, use that instead -VM_COUNT=${VM_COUNT:-1} -NET_NAME=doctor_net -NET_CIDR=192.168.168.0/24 -ALARM_BASENAME=doctor_alarm -CONSUMER_PORT=12346 -DOCTOR_USER=doctor -DOCTOR_PW=doctor -DOCTOR_PROJECT=doctor -DOCTOR_ROLE=_member_ -PROFILER_TYPE=${PROFILER_TYPE:-poc} -PYTHON_ENABLE=${PYTHON_ENABLE:-false} - -TOP_DIR=$(cd $(dirname "$0") && pwd) - -as_doctor_user="--os-username $DOCTOR_USER --os-password $DOCTOR_PW - --os-project-name $DOCTOR_PROJECT --os-tenant-name $DOCTOR_PROJECT" -# NOTE: ceilometer command still requires '--os-tenant-name'. -#ceilometer="ceilometer ${as_doctor_user/--os-project-name/--os-tenant-name}" -ceilometer="ceilometer $as_doctor_user" -as_admin_user="--os-username admin --os-project-name $DOCTOR_PROJECT - --os-tenant-name $DOCTOR_PROJECT" - -upper_constraints="https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/ocata" -pip_install="pip install -c${upper_constraints}" - -# Functions - -get_compute_host_info() { - # get computer host info which first VM boot in as admin user - COMPUTE_HOST=$(openstack $as_admin_user server show ${VM_BASENAME}1 | - grep "OS-EXT-SRV-ATTR:host " | awk '{ print $4 }') - compute_host_in_undercloud=${COMPUTE_HOST%%.*} - die_if_not_set $LINENO COMPUTE_HOST "Failed to get compute hostname" - - get_compute_ip_from_hostname $COMPUTE_HOST - - echo "COMPUTE_HOST=$COMPUTE_HOST" - echo "COMPUTE_IP=$COMPUTE_IP" - - # verify connectivity to target compute host - ping -c 1 "$COMPUTE_IP" - if [[ $? -ne 0 ]] ; then - die $LINENO "Can not ping to computer host" - fi - - # verify ssh to target compute host - ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'exit' - if [[ $? -ne 0 ]] ; then - die $LINENO "Can not ssh to computer host" - fi -} - -# TODO(r-mibu): update this function to support consumer instance -# and migrate this function into installer lib -get_consumer_ip___to_be_removed() { - local get_consumer_command="ip route get $COMPUTE_IP | awk '/ src /{print \$NF}'" - if is_installer apex; then - CONSUMER_IP=$(sudo ssh $ssh_opts root@$INSTALLER_IP \ - "$get_consumer_command") - elif is_installer fuel; then - CONSUMER_IP=$(sudo sshpass -p r00tme ssh $ssh_opts root@${INSTALLER_IP} \ - "$get_consumer_command") - elif is_installer local; then - CONSUMER_IP=`$get_consumer_command` - fi - echo "CONSUMER_IP=$CONSUMER_IP" - - die_if_not_set $LINENO CONSUMER_IP "Could not get CONSUMER_IP." -} - -download_image() { - #if a different name was provided for the image in the enviroment there's no need to download the image - use_existing_image=false - openstack image list | grep -q " $IMAGE_NAME " && use_existing_image=true - - if [[ "$use_existing_image" == false ]] ; then - [ -e "$IMAGE_FILE" ] && return 0 - wget "$IMAGE_URL" -o "$IMAGE_FILE" - fi -} - -register_image() { - openstack image list | grep -q " $IMAGE_NAME " && return 0 - openstack image create "$IMAGE_NAME" \ - --public \ - --disk-format "$IMAGE_FORMAT" \ - --container-format bare \ - --file "$IMAGE_FILE" -} - -create_test_user() { - openstack project list | grep -q " $DOCTOR_PROJECT " || { - openstack project create --description "Doctor Project" \ - "$DOCTOR_PROJECT" - } - openstack user list | grep -q " $DOCTOR_USER " || { - openstack user create "$DOCTOR_USER" --password "$DOCTOR_PW" \ - --project "$DOCTOR_PROJECT" - } - openstack role show "$DOCTOR_ROLE" | grep -q " $DOCTOR_ROLE " || { - openstack role create "$DOCTOR_ROLE" - } - openstack role assignment list --user "$DOCTOR_USER" \ - --project "$DOCTOR_PROJECT" --names | grep -q " $DOCTOR_ROLE " || { - openstack role add "$DOCTOR_ROLE" --user "$DOCTOR_USER" \ - --project "$DOCTOR_PROJECT" - } - openstack role assignment list --user admin --project "$DOCTOR_PROJECT" \ - --names | grep -q " admin " || { - openstack role add admin --user admin --project "$DOCTOR_PROJECT" - } - # tojuvone: openstack quota show is broken and have to use nova - # https://bugs.launchpad.net/manila/+bug/1652118 - # Note! while it is encouraged to use openstack client it has proven - # quite buggy. - # QUOTA=$(openstack quota show $DOCTOR_PROJECT) - DOCTOR_QUOTA=$(nova quota-show --tenant $DOCTOR_PROJECT) - # We make sure that quota allows number of instances and cores - OLD_INSTANCE_QUOTA=$(echo "${DOCTOR_QUOTA}" | grep " instances " | \ - awk '{print $4}') - if [ $OLD_INSTANCE_QUOTA -lt $VM_COUNT ]; then - openstack quota set --instances $VM_COUNT \ - $DOCTOR_USER - fi - OLD_CORES_QUOTA=$(echo "${DOCTOR_QUOTA}" | grep " cores " | \ - awk '{print $4}') - if [ $OLD_CORES_QUOTA -lt $VM_COUNT ]; then - openstack quota set --cores $VM_COUNT \ - $DOCTOR_USER - fi -} - -remove_test_user() { - openstack project list | grep -q " $DOCTOR_PROJECT " && { - openstack role assignment list --user admin \ - --project "$DOCTOR_PROJECT" --names | grep -q " admin " && { - openstack role remove admin --user admin --project "$DOCTOR_PROJECT" - } - openstack user list | grep -q " $DOCTOR_USER " && { - openstack role assignment list --user "$DOCTOR_USER" \ - --project "$DOCTOR_PROJECT" --names | grep -q " $DOCTOR_ROLE " && { - openstack role remove "$DOCTOR_ROLE" --user "$DOCTOR_USER" \ - --project "$DOCTOR_PROJECT" - } - openstack user delete "$DOCTOR_USER" - } - openstack project delete "$DOCTOR_PROJECT" - } -} - -boot_vm() { - # test VM done with test user, so can test non-admin - - if ! openstack $as_doctor_user network show $NET_NAME; then - openstack $as_doctor_user network create $NET_NAME - fi - if ! openstack $as_doctor_user subnet show $NET_NAME; then - openstack $as_doctor_user subnet create $NET_NAME \ - --network $NET_NAME --subnet-range $NET_CIDR --no-dhcp - fi - net_id=$(openstack $as_doctor_user network show $NET_NAME -f value -c id) - - servers=$(openstack $as_doctor_user server list) - for i in `seq $VM_COUNT`; do - echo "${servers}" | grep -q " $VM_BASENAME$i " && continue - openstack $as_doctor_user server create --flavor "$VM_FLAVOR" \ - --image "$IMAGE_NAME" --nic net-id=$net_id "$VM_BASENAME$i" - done - sleep 1 -} - -create_alarm() { - # get vm_id as test user - alarm_list=$($ceilometer alarm-list) - vms=$(openstack $as_doctor_user server list) - for i in `seq $VM_COUNT`; do - echo "${alarm_list}" | grep -q " $ALARM_BASENAME$i " || { - vm_id=$(echo "${vms}" | grep " $VM_BASENAME$i " | awk '{print $2}') - # TODO(r-mibu): change notification endpoint from localhost to the - # consumer. IP address (functest container). - $ceilometer alarm-event-create \ - --name "$ALARM_BASENAME$i" \ - --alarm-action "http://localhost:$CONSUMER_PORT/failure" \ - --description "VM failure" \ - --enabled True \ - --repeat-actions False \ - --severity "moderate" \ - --event-type compute.instance.update \ - -q "traits.state=string::error; \ - traits.instance_id=string::$vm_id" - } - done -} - -start_consumer() { - pgrep -f "python consumer.py" && return 0 - python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 & - - # NOTE(r-mibu): create tunnel to the controller nodes, so that we can - # avoid some network problems dpends on infra and installers. - # This tunnel will be terminated by stop_consumer() or after 10 mins passed. - if ! is_installer local; then - for ip in $CONTROLLER_IPS - do - forward_rule="-R $CONSUMER_PORT:localhost:$CONSUMER_PORT" - tunnel_command="sudo ssh $ssh_opts_cpu $COMPUTE_USER@$ip $forward_rule sleep 600" - $tunnel_command > "ssh_tunnel.${ip}.log" 2>&1 < /dev/null & - done - fi -} - -stop_consumer() { - pgrep -f "python consumer.py" || return 0 - kill $(pgrep -f "python consumer.py") - - # NOTE(r-mibu): terminate tunnels to the controller nodes - if ! is_installer local; then - for ip in $CONTROLLER_IPS - do - forward_rule="-R $CONSUMER_PORT:localhost:$CONSUMER_PORT" - tunnel_command="sudo ssh $ssh_opts_cpu $COMPUTE_USER@$ip $forward_rule sleep 600" - kill $(pgrep -f "$tunnel_command") - done - fi -} - -wait_for_vm_launch() { - echo "waiting for vm launch..." - - count=0 - while [[ ${count} -lt 60 ]] - do - active_count=0 - vms=$(openstack $as_doctor_user server list) - for i in `seq $VM_COUNT`; do - state=$(echo "${vms}" | grep " $VM_BASENAME$i " | awk '{print $6}') - if [[ "$state" == "ACTIVE" ]]; then - active_count=$(($active_count+1)) - elif [[ "$state" == "ERROR" ]]; then - die $LINENO "vm state $VM_BASENAME$i is ERROR" - else - #This VM not yet active - count=$(($count+1)) - sleep 5 - continue - fi - done - [[ $active_count -eq $VM_COUNT ]] && { - echo "get computer host info..." - get_compute_host_info - VMS_ON_FAILED_HOST=$(openstack $as_doctor_user server list --host \ - $COMPUTE_HOST | grep " ${VM_BASENAME}" | wc -l) - return 0 - } - #Not all VMs active - count=$(($count+1)) - sleep 5 - done - die $LINENO "Time out while waiting for VM launch" -} - -inject_failure() { - echo "disabling network of compute host [$COMPUTE_HOST] for 3 mins..." - cat > disable_network.sh << 'END_TXT' -#!/bin/bash -x -sleep 1 -if [ -n "@INTERFACE_NAME@" ]; then - dev=@INTERFACE_NAME@ -else - dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}') -fi -sudo ip link set $dev down -echo "doctor set link down at" $(date "+%s.%N") -sleep 180 -sudo ip link set $dev up -sleep 1 -END_TXT - sed -i -e "s/@COMPUTE_IP@/$COMPUTE_IP/" disable_network.sh - sed -i -e "s/@INTERFACE_NAME@/$INTERFACE_NAME/" disable_network.sh - chmod +x disable_network.sh - scp $ssh_opts_cpu disable_network.sh "$COMPUTE_USER@$COMPUTE_IP:" - ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &' - # use host time to get rid of potential time sync deviation between nodes - triggered=$(date "+%s.%N") -} - -wait_consumer() { - local interval=1 - local rounds=$(($1 / $interval)) - for i in `seq $rounds`; do - notified_count=$(grep "doctor consumer notified at" consumer.log | wc -l) - if [[ $notified_count -eq $VMS_ON_FAILED_HOST ]]; then - return 0 - fi - sleep $interval - done - die $LINENO "Consumer hasn't received fault notification." -} - -calculate_notification_time() { - wait_consumer 60 - #keep 'at' as the last keyword just before the value, and - #use regex to get value instead of the fixed column - if [ ! -f monitor.log ]; then - scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:monitor.log" . - fi - detected=$(grep "doctor monitor detected at" monitor.log |\ - sed -e "s/^.* at //" | tail -1) - notified=$(grep "doctor consumer notified at" consumer.log |\ - sed -e "s/^.* at //" | tail -1) - - echo "$notified $detected" | \ - awk '{ - d = $1 - $2; - if (d < 1 && d > 0) { print d " OK"; exit 0 } - else { print d " NG"; exit 1 } - }' -} - -check_host_status() { - # Check host related to first Doctor VM is in wanted state - # $1 Expected state - # $2 Seconds to wait to have wanted state - expected_state=$1 - local interval=5 - local rounds=$(($2 / $interval)) - for i in `seq $rounds`; do - host_status_line=$(openstack $as_doctor_user --os-compute-api-version \ - 2.16 server show ${VM_BASENAME}1 | grep "host_status") - host_status=$(echo $host_status_line | awk '{print $4}') - die_if_not_set $LINENO host_status "host_status not reported by: nova show ${VM_BASENAME}1" - if [[ "$expected_state" =~ "$host_status" ]] ; then - echo "${VM_BASENAME}1 showing host_status: $host_status" - return 0 - else - sleep $interval - fi - done - if [[ "$expected_state" =~ "$host_status" ]] ; then - echo "${VM_BASENAME}1 showing host_status: $host_status" - else - die $LINENO "host_status:$host_status not equal to expected_state: $expected_state" - fi -} - -unset_forced_down_hosts() { - # for debug - openstack compute service list --service nova-compute - - downed_computes=$(openstack compute service list --service nova-compute \ - -f value -c Host -c State | grep ' down$' \ - | sed -e 's/ *down$//') - echo "downed_computes: $downed_computes" - for host in $downed_computes - do - # TODO(r-mibu): use openstack client - #openstack compute service set --up $host nova-compute - nova service-force-down --unset $host nova-compute - done - - echo "waiting disabled compute host back to be enabled..." - wait_until 'openstack compute service list --service nova-compute - -f value -c State | grep -q down' 240 5 - - for host in $downed_computes - do - # TODO(r-mibu): improve 'get_compute_ip_from_hostname' - get_compute_ip_from_hostname $host - wait_until "! ping -c 1 $COMPUTE_IP" 120 5 - done -} - -collect_logs() { - if [[ -n "$COMPUTE_IP" ]];then - scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" . - fi - - # TODO(yujunz) collect other logs, e.g. nova, aodh -} - -run_profiler() { - if [[ "$PROFILER_TYPE" == "poc" ]]; then - linkdown=$(grep "doctor set link down at " disable_network.log |\ - sed -e "s/^.* at //") - vmdown=$(grep "doctor mark vm.* error at" inspector.log |tail -n 1 |\ - sed -e "s/^.* at //") - hostdown=$(grep "doctor mark host.* down at" inspector.log |\ - sed -e "s/^.* at //") - - # TODO(yujunz) check the actual delay to verify time sync status - # expected ~1s delay from $trigger to $linkdown - relative_start=${linkdown} - export DOCTOR_PROFILER_T00=$(python -c \ - "print(int(($linkdown-$relative_start)*1000))") - export DOCTOR_PROFILER_T01=$(python -c \ - "print(int(($detected-$relative_start)*1000))") - export DOCTOR_PROFILER_T03=$(python -c \ - "print(int(($vmdown-$relative_start)*1000))") - export DOCTOR_PROFILER_T04=$(python -c \ - "print(int(($hostdown-$relative_start)*1000))") - export DOCTOR_PROFILER_T09=$(python -c \ - "print(int(($notified-$relative_start)*1000))") - - python profiler_poc.py > doctor_profiler.log 2>&1 - fi -} - -cleanup() { - set +e - echo "cleanup..." - stop_inspector - stop_consumer - - unset_forced_down_hosts - stop_monitor - collect_logs - - vms=$(openstack $as_doctor_user server list) - vmstodel="" - for i in `seq $VM_COUNT`; do - $(echo "${vms}" | grep -q " $VM_BASENAME$i ") && - vmstodel+=" $VM_BASENAME$i" - done - [[ $vmstodel ]] && openstack $as_doctor_user server delete $vmstodel - alarm_list=$($ceilometer alarm-list) - for i in `seq $VM_COUNT`; do - alarm_id=$(echo "${alarm_list}" | grep " $ALARM_BASENAME$i " | - awk '{print $2}') - [ -n "$alarm_id" ] && $ceilometer alarm-delete "$alarm_id" - done - openstack $as_doctor_user subnet delete $NET_NAME - sleep 1 - openstack $as_doctor_user network delete $NET_NAME - sleep 1 - - image_id=$(openstack image list | grep " $IMAGE_NAME " | awk '{print $2}') - sleep 1 - #if an existing image was used, there's no need to remove it here - if [[ "$use_existing_image" == false ]] ; then - [ -n "$image_id" ] && openstack image delete "$image_id" - fi - - remove_test_user - - cleanup_installer - cleanup_inspector - cleanup_monitor - - # NOTE: Temporal log printer. - for f in $(find . -name '*.log') - do - echo - echo "[$f]" - sed -e 's/^/ | /' $f - echo - done -} - -setup_python_packages() { - pip freeze |grep -i flask\= > /dev/null || sudo ${pip_install} flask - command -v openstack || sudo ${pip_install} python-openstackclient - command -v ceilometer || sudo ${pip_install} python-ceilometerclient - command -v congress || sudo ${pip_install} python-congressclient -} - -# Main process - -if [[ $PYTHON_ENABLE == [Tt]rue ]]; then - which tox || sudo ${pip_install} tox - if [ -f /usr/bin/apt-get ]; then - sudo apt-get install -y python3-dev - elif [ -f /usr/bin/yum ] ; then - sudo yum install -y python3-devel - fi - - cd $TOP_DIR - echo "executing tox..." - tox - exit $? -fi - -echo "Note: doctor/tests/run.sh has been executed." -git log --oneline -1 || true # ignore even you don't have git installed - -trap cleanup EXIT - -setup_python_packages - -source $TOP_DIR/functions-common -source $TOP_DIR/lib/installer -source $TOP_DIR/lib/inspector -source $TOP_DIR/lib/monitor - -rm -f *.log - -setup_installer - -echo "preparing VM image..." -download_image -register_image - -echo "creating test user..." -create_test_user - -echo "creating VM..." -boot_vm -wait_for_vm_launch - -echo "creating alarm..." -#TODO: change back to use, network problems depends on infra and installers -#get_consumer_ip -create_alarm - -echo "starting doctor sample components..." -start_inspector -start_monitor -start_consumer - -sleep 60 -echo "injecting host failure..." -inject_failure - -check_host_status "(DOWN|UNKNOWN)" 60 -unset_forced_down_hosts -calculate_notification_time -collect_logs -run_profiler - -echo "done" |