diff options
| -rw-r--r-- | sdnvpn/lib/gather_logs.py | 54 | ||||
| -rwxr-xr-x | sdnvpn/sh_utils/fetch-log-script.sh | 163 | ||||
| -rw-r--r-- | sdnvpn/test/functest/run_tests.py | 8 |
3 files changed, 224 insertions, 1 deletions
diff --git a/sdnvpn/lib/gather_logs.py b/sdnvpn/lib/gather_logs.py new file mode 100644 index 0000000..cf84f43 --- /dev/null +++ b/sdnvpn/lib/gather_logs.py @@ -0,0 +1,54 @@ +# Copyright (c) 2017 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# performes: +# - Gather +# - odl logs +# - ovs logs +# - neutron logs +# - odl datastore state +# - ovs state (openflow and dp) +# - optional - memory dump from odl + +import os +import inspect + +import sdnvpn.lib.utils as test_utils +import functest.utils.functest_utils as ft_utils +from functest.utils.constants import CONST + +LIB_PATH = os.path.dirname( + os.path.abspath(inspect.getfile(inspect.currentframe()))) + + +def gather_logs(name): + openstack_nodes = test_utils.get_nodes() + + ft_utils.execute_command_raise('rm -rf /tmp/sdnvpn-logs/;' + 'mkdir -p /tmp/sdnvpn-logs/') + for node in openstack_nodes: + node.put_file('%s/../sh_utils/fetch-log-script.sh' + % LIB_PATH, + '/tmp/fetch-log-script.sh') + node.run_cmd('sudo bash /tmp/fetch-log-script.sh') + node.get_file('/tmp/log_output.tar.gz', + '/tmp/log_output-%s.tar.gz' + % node.get_dict()['name']) + ft_utils.execute_command_raise('mkdir -p /tmp/sdnvpn-logs/') + ft_utils.execute_command_raise( + 'cd /tmp/sdnvpn-logs/; ' + 'tar -xzvf /tmp/log_output-%s.tar.gz --strip-components=1' + % node.get_dict()['name']) + + ft_utils.execute_command_raise('cd %s;tar czvf sdnvpn-logs-%s.tar.gz' + '-C /tmp/ sdnvpn-logs/' + % (CONST.__getattribute__('dir_results'), + name)) + + +if __name__ == '__main__': + gather_logs('test') diff --git a/sdnvpn/sh_utils/fetch-log-script.sh b/sdnvpn/sh_utils/fetch-log-script.sh new file mode 100755 index 0000000..8e825d0 --- /dev/null +++ b/sdnvpn/sh_utils/fetch-log-script.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +### Configuration Required +set -e + +tmp_folder=/tmp/opnfv-logs-$HOSTNAME/ +rm -rf $tmp_folder +mkdir -p $tmp_folder + +if [ "`whoami`" != "root" ]; then + echo "You need to be root" + exit 1 +fi +#install needed packages +if ! which sshpass 2>&1 >> /dev/null;then + yum install -y sshpass &> /dev/null || apt-get install -y sshpass &> /dev/null +fi +################# DPN Stauts ####################################################### + +############### Enable TRACE for org.opendaylight.vpnservice,Configuring KARAF file size to 50MB and clear OLD Logs + +_enable_logs(){ + level=$1 + lib=$2 + echo "setting log level from $lib to $level" + $KARAF "log:set $level $lib" 2>&1 +} +logs_enable() +{ + set_to_trace=( org.opendaylight.vpnservice org.opendaylight.bgpmanager org.opendaylight.vpnservice.dhcpservice org.opendaylight.netvirt org.opendaylight.netvirt.aclservice org.opendaylight.neutron org.opendaylight.netvirt.neutronvpn org.opendaylight.neutron ) + set_to_error=( org.opendaylight.openflowplugin ) + set_to_warn=( org.opendaylight.vpnservice.arputil org.opendaylight.vpnservice.alivenessmonitor org.opendaylight.vpnservice.interfacemgr.pmcounters org.opendaylight.genius.arputil org.opendaylight.genius.interfacemgr org.opendaylight.genius.alivenessmonitor ) + echo + echo + echo "===================" + echo "Enabled logs in ODL" + echo "===================" + echo + + for lib in $set_to_trace;do + _enable_logs TRACE $lib + done + for lib in $set_to_error;do + _enable_logs ERROR $lib + done + for lib in $set_to_warn;do + _enable_logs WARN $lib + done + sleep 1 + echo "Removing existing karaf logs............" + rm -rf /opt/opendaylight/data/log/karaf.log.* + echo "Modifying karaf log file size to 50MB............" + sed -i -e 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=50MB/g' /opt/opendaylight/etc/org.ops4j.pax.logging.cfg + echo "Truncating karaf logs............" + echo "truncated" >/opt/sdnc/opendaylight/data/log/karaf.log +} + +logs_disable() +{ + set_to_info=( org.opendaylight.vpnservice org.opendaylight.bgpmanager org.opendaylight.vpnservice.dhcpservice org.opendaylight.netvirt org.opendaylight.netvirt.aclservice org.opendaylight.neutron org.opendaylight.netvirt.neutronvpn org.opendaylight.neutron org.opendaylight.openflowplugin org.opendaylight.vpnservice.arputil org.opendaylight.vpnservice.alivenessmonitor org.opendaylight.vpnservice.interfacemgr.pmcounters org.opendaylight.genius.arputil org.opendaylight.genius.interfacemgr org.opendaylight.genius.alivenessmonitor ) + echo + echo + echo "===========================" + echo "set log to INFO logs in ODL" + echo "===========================" + echo + + for lib in $set_to_info;do + _enable_logs INFO $lib + done + +} + +log_command_exec(){ + file=$1 + touch $file + shift + cmd=$@ + echo "==========================================================" >> $file + echo "$cmd" >> $file + echo "==========================================================" >> $file + $cmd 2>&1 >> $file +} + +flows() +{ + flows=$tmp_folder/DPN-flows-groups-$HOSTNAME.txt + log_command_exec "$flows" ovs-vsctl show + log_command_exec "$flows" ovs-ofctl -O OpenFlow13 dump-flows br-int + log_command_exec "$flows" ovs-ofctl -O OpenFlow13 dump-groups br-int + log_command_exec "$flows" ovs-ofctl show br-int -O OpenFlow13 +} + +node(){ + node=$tmp_folder/$HOSTNAME.txt + log_command_exec "$node" ifconfig -a + files_folders=( /opt/opendaylight/data/log/ /var/log/openvswitch/ /var/log/neutron/) + for ((i = 0; i < ${#files_folders[@]};i++));do + if [ -e ${files_folders[$i]} ];then + cp -r ${files_folders[$i]} $tmp_folder/ + fi + done + # not all messages only tail the last 10k lines + tail -n 10000 /var/log/messages > messages +} + +_curl_data_store(){ + file=$1 + url=$2 + touch $file + echo "============================= $url ======================" >> $file + curl --silent -u admin:admin -X GET http://$odl_ip_port/$url | python -mjson.tool 2>&1 >> $file +} + +_get_output_karaf(){ + file=$1 + touch $file + shift + echo "============================ KARAF $@ ===================" >> $file + sshpass -p karaf ssh -p 8101 -o "StrictHostKeyChecking no" karaf@localhost "$@" 2>&1 >> $file + +} +datastore() +{ + + dump=$tmp_folder/dump-$HOSTNAME.txt + operational=$tmp_folder/Operational-Inventory-$HOSTNAME.txt + karaf_output=$tmp_folder/Karaf_out-$HOSTNAME.txt + odl_ip_port=$(grep ^url= /etc/neutron/plugins/ml2/ml2_conf.ini |cut -d '/' -f3) + + config_urls=( restconf/config/neutron:neutron/networks/ restconf/config/neutron:neutron/subnets/ restconf/config/neutron:neutron/ports/ restconf/config/neutron:neutron/routers/ restconf/config/itm:transport-zones/ restconf/config/itm-state:tunnels_state/ restconf/config/itm-state:external-tunnel-list/ restconf/config/itm-state:dpn-endpoints/ restconf/config/itm-config:vtep-config-schemas/ restconf/config/itm-config:tunnel-monitor-enabled/ restconf/config/itm-config:tunnel-monitor-interval/ restconf/config/interface-service-bindings:service-bindings/ restconf/config/l3vpn:vpn-instances/ restconf/config/ietf-interfaces:interfaces/ restconf/config/l3vpn:vpn-interfaces/ restconf/config/odl-fib:fibEntries restconf/config/neutronvpn:networkMaps restconf/config/neutronvpn:subnetmaps restconf/config/neutronvpn:vpnMaps restconf/config/neutronvpn:neutron-port-data restconf/config/id-manager:id-pools/ restconf/config/elan:elan-instances/ restconf/config/elan:elan-interfaces/ restconf/config/elan:elan-state/ restconf/config/elan:elan-forwarding-tables/ restconf/config/elan:elan-interface-forwarding-entries/ restconf/config/elan:elan-dpn-interfaces/ restconf/config/elan:elan-tag-name-map/ restconf/config/odl-nat:external-networks/ restconf/config/odl-nat:ext-routers/ restconf/config/odl-nat:intext-ip-port-map/ restconf/config/odl-nat:snatint-ip-port-map/ restconf/config/odl-l3vpn:vpn-instance-to-vpn-id/ restconf/config/neutronvpn:neutron-router-dpns/ restconf/operational/itm-config:tunnel-monitor-interval/ restconf/config/itm-config:tunnel-monitor-interval/ restconf/operational/itm-config:tunnel-monitor-params/ restconf/config/itm-config:tunnel-monitor-params/ restconf/config/vpnservice-dhcp:designated-switches-for-external-tunnels/ restconf/config/neutron:neutron/security-groups/ restconf/config/neutron:neutron/security-rules/ restconf/config/network-topology:network-topology/topology/hwvtep:1 restconf/config/network-topology:network-topology/topology/ovsdb:1 ) + + for url in ${config_urls[@]};do + _curl_data_store $dump $url + done + + operational_urls=( restconf/operational/itm-state:tunnels_state/ restconf/operational/odl-interface-meta:bridge-ref-info/ restconf/operational/odl-l3vpn:prefix-to-interface/ restconf/operational/odl-l3vpn:vpn-instance-op-data/ restconf/operational/l3vpn:vpn-interfaces/ restconf/operational/ietf-interfaces:interfaces-state/ restconf/operational/odl-l3vpn:prefix-to-interface/ restconf/operational/l3nexthop:l3nexthop restconf/operational/elan:elan-instances/ restconf/operational/elan:elan-interfaces/ restconf/operational/elan:elan-state/ restconf/operational/elan:elan-forwarding-tables/ restconf/operational/elan:elan-interface-forwarding-entries/ restconf/operational/elan:elan-dpn-interfaces/ restconf/operational/elan:elan-tag-name-map/ restconf/operational/odl-nat:napt-switches/ restconf/operational/odl-nat:intext-ip-map/ restconf/operational/odl-nat:external-ips-counter/ restconf/operational/neutronvpn:neutron-router-dpns/ restconf/operational/odl-l3vpn:subnet-op-data/ restconf/operational/odl-l3vpn:port-op-data/ restconf/operational/odl-l3vpn:vpn-to-extraroute restconf/operational/neutronvpn:neutron-vpn-portip-port-data/ restconf/operational/odl-fib:label-route-map/ restconf/operational/itm-config:tunnel-monitor-interval/ restconf/operational/itm-config:tunnel-monitor-params/ restconf/operational/odl-interface-meta:if-indexes-interface-map/ restconf/operational/network-topology:network-topology/topology/hwvtep:1 restconf/operational/network-topology:network-topology/topology/ovsdb:1 restconf/operational/opendaylight-inventory:nodes/ restconf/operational/opendaylight-inventory:nodes/ ) + + for url in ${operational_urls[@]};do + _curl_data_store $operational $url + done + + + karaf_commands=( "display-bgp-config --debug" "bundle:list -s" fib-show "show-bgp --cmd 'ip bgp summary'" "show-bgp --cmd 'ip bgp neighbor'" "show-bgp --cmd 'ip bgp vpnv4 all summary'") + for ((i = 0; i < ${#karaf_commands[@]}; i++));do + _get_output_karaf $karaf_output ${karaf_commands[$i]} + done + # TODO check this one: + # curl --silent -u admin:admin -H "Content-Type: application/json" --data "{"input" : { } }" -X POST http://$OS_MIP:8181/restconf/operations/neutronvpn:getL3VPN | python -mjson.tool >>$file1 +} + +# check if running on karaf node +if netstat -atnp |grep 8101 -q;then + datastore +fi +flows +node + +pushd /tmp +rm -rf log_output.tar.gz +#tar -czvf dump_output-$HOSTNAME-`date +"%d-%m-%H-%M"`.tar.gz $tmp_folder +tar -czvf log_output.tar.gz $tmp_folder 2>&1 + diff --git a/sdnvpn/test/functest/run_tests.py b/sdnvpn/test/functest/run_tests.py index a861a47..1bffe67 100644 --- a/sdnvpn/test/functest/run_tests.py +++ b/sdnvpn/test/functest/run_tests.py @@ -19,7 +19,7 @@ import yaml import functest.utils.functest_utils as ft_utils from sdnvpn.lib import config as sdnvpn_config - +from sdnvpn.lib.gather_logs import gather_logs logger = logging.getLogger('sdnvpn-run-tests') @@ -93,6 +93,12 @@ def main(report=False): push_results( test_name_db, start_time, end_time, status, details) + try: + gather_logs('overall') + except Exception as ex: + logger.error(('Something went wrong in the Log gathering.' + 'Ex: Trace: %s') + % ex, traceback.format_exc()) if overall_status == "FAIL": sys.exit(-1) |