summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNikolas Hermanns <nikolas.hermanns@ericsson.com>2017-07-12 12:07:46 +0000
committerNikolas Hermanns <nikolas.hermanns@ericsson.com>2017-07-12 13:20:51 +0000
commit3668645d06669e0abe44535ee3a4450a0b83709c (patch)
treecf87a89261546156d64d1cbe6943efade9687983
parent74db571092ab8650d8059430ea4e4033d2269527 (diff)
Adding log gathering
Change-Id: I0f13e5aa8aea25e549c9ffede63085a86153cdba Signed-off-by: Nikolas Hermanns <nikolas.hermanns@ericsson.com>
-rw-r--r--sdnvpn/lib/gather_logs.py54
-rwxr-xr-xsdnvpn/sh_utils/fetch-log-script.sh163
-rw-r--r--sdnvpn/test/functest/run_tests.py8
3 files changed, 224 insertions, 1 deletions
diff --git a/sdnvpn/lib/gather_logs.py b/sdnvpn/lib/gather_logs.py
new file mode 100644
index 0000000..cf84f43
--- /dev/null
+++ b/sdnvpn/lib/gather_logs.py
@@ -0,0 +1,54 @@
+# Copyright (c) 2017 All rights reserved
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# performes:
+# - Gather
+# - odl logs
+# - ovs logs
+# - neutron logs
+# - odl datastore state
+# - ovs state (openflow and dp)
+# - optional - memory dump from odl
+
+import os
+import inspect
+
+import sdnvpn.lib.utils as test_utils
+import functest.utils.functest_utils as ft_utils
+from functest.utils.constants import CONST
+
+LIB_PATH = os.path.dirname(
+ os.path.abspath(inspect.getfile(inspect.currentframe())))
+
+
+def gather_logs(name):
+ openstack_nodes = test_utils.get_nodes()
+
+ ft_utils.execute_command_raise('rm -rf /tmp/sdnvpn-logs/;'
+ 'mkdir -p /tmp/sdnvpn-logs/')
+ for node in openstack_nodes:
+ node.put_file('%s/../sh_utils/fetch-log-script.sh'
+ % LIB_PATH,
+ '/tmp/fetch-log-script.sh')
+ node.run_cmd('sudo bash /tmp/fetch-log-script.sh')
+ node.get_file('/tmp/log_output.tar.gz',
+ '/tmp/log_output-%s.tar.gz'
+ % node.get_dict()['name'])
+ ft_utils.execute_command_raise('mkdir -p /tmp/sdnvpn-logs/')
+ ft_utils.execute_command_raise(
+ 'cd /tmp/sdnvpn-logs/; '
+ 'tar -xzvf /tmp/log_output-%s.tar.gz --strip-components=1'
+ % node.get_dict()['name'])
+
+ ft_utils.execute_command_raise('cd %s;tar czvf sdnvpn-logs-%s.tar.gz'
+ '-C /tmp/ sdnvpn-logs/'
+ % (CONST.__getattribute__('dir_results'),
+ name))
+
+
+if __name__ == '__main__':
+ gather_logs('test')
diff --git a/sdnvpn/sh_utils/fetch-log-script.sh b/sdnvpn/sh_utils/fetch-log-script.sh
new file mode 100755
index 0000000..8e825d0
--- /dev/null
+++ b/sdnvpn/sh_utils/fetch-log-script.sh
@@ -0,0 +1,163 @@
+#!/bin/bash
+
+### Configuration Required
+set -e
+
+tmp_folder=/tmp/opnfv-logs-$HOSTNAME/
+rm -rf $tmp_folder
+mkdir -p $tmp_folder
+
+if [ "`whoami`" != "root" ]; then
+ echo "You need to be root"
+ exit 1
+fi
+#install needed packages
+if ! which sshpass 2>&1 >> /dev/null;then
+ yum install -y sshpass &> /dev/null || apt-get install -y sshpass &> /dev/null
+fi
+################# DPN Stauts #######################################################
+
+############### Enable TRACE for org.opendaylight.vpnservice,Configuring KARAF file size to 50MB and clear OLD Logs
+
+_enable_logs(){
+ level=$1
+ lib=$2
+ echo "setting log level from $lib to $level"
+ $KARAF "log:set $level $lib" 2>&1
+}
+logs_enable()
+{
+ set_to_trace=( org.opendaylight.vpnservice org.opendaylight.bgpmanager org.opendaylight.vpnservice.dhcpservice org.opendaylight.netvirt org.opendaylight.netvirt.aclservice org.opendaylight.neutron org.opendaylight.netvirt.neutronvpn org.opendaylight.neutron )
+ set_to_error=( org.opendaylight.openflowplugin )
+ set_to_warn=( org.opendaylight.vpnservice.arputil org.opendaylight.vpnservice.alivenessmonitor org.opendaylight.vpnservice.interfacemgr.pmcounters org.opendaylight.genius.arputil org.opendaylight.genius.interfacemgr org.opendaylight.genius.alivenessmonitor )
+ echo
+ echo
+ echo "==================="
+ echo "Enabled logs in ODL"
+ echo "==================="
+ echo
+
+ for lib in $set_to_trace;do
+ _enable_logs TRACE $lib
+ done
+ for lib in $set_to_error;do
+ _enable_logs ERROR $lib
+ done
+ for lib in $set_to_warn;do
+ _enable_logs WARN $lib
+ done
+ sleep 1
+ echo "Removing existing karaf logs............"
+ rm -rf /opt/opendaylight/data/log/karaf.log.*
+ echo "Modifying karaf log file size to 50MB............"
+ sed -i -e 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=50MB/g' /opt/opendaylight/etc/org.ops4j.pax.logging.cfg
+ echo "Truncating karaf logs............"
+ echo "truncated" >/opt/sdnc/opendaylight/data/log/karaf.log
+}
+
+logs_disable()
+{
+ set_to_info=( org.opendaylight.vpnservice org.opendaylight.bgpmanager org.opendaylight.vpnservice.dhcpservice org.opendaylight.netvirt org.opendaylight.netvirt.aclservice org.opendaylight.neutron org.opendaylight.netvirt.neutronvpn org.opendaylight.neutron org.opendaylight.openflowplugin org.opendaylight.vpnservice.arputil org.opendaylight.vpnservice.alivenessmonitor org.opendaylight.vpnservice.interfacemgr.pmcounters org.opendaylight.genius.arputil org.opendaylight.genius.interfacemgr org.opendaylight.genius.alivenessmonitor )
+ echo
+ echo
+ echo "==========================="
+ echo "set log to INFO logs in ODL"
+ echo "==========================="
+ echo
+
+ for lib in $set_to_info;do
+ _enable_logs INFO $lib
+ done
+
+}
+
+log_command_exec(){
+ file=$1
+ touch $file
+ shift
+ cmd=$@
+ echo "==========================================================" >> $file
+ echo "$cmd" >> $file
+ echo "==========================================================" >> $file
+ $cmd 2>&1 >> $file
+}
+
+flows()
+{
+ flows=$tmp_folder/DPN-flows-groups-$HOSTNAME.txt
+ log_command_exec "$flows" ovs-vsctl show
+ log_command_exec "$flows" ovs-ofctl -O OpenFlow13 dump-flows br-int
+ log_command_exec "$flows" ovs-ofctl -O OpenFlow13 dump-groups br-int
+ log_command_exec "$flows" ovs-ofctl show br-int -O OpenFlow13
+}
+
+node(){
+ node=$tmp_folder/$HOSTNAME.txt
+ log_command_exec "$node" ifconfig -a
+ files_folders=( /opt/opendaylight/data/log/ /var/log/openvswitch/ /var/log/neutron/)
+ for ((i = 0; i < ${#files_folders[@]};i++));do
+ if [ -e ${files_folders[$i]} ];then
+ cp -r ${files_folders[$i]} $tmp_folder/
+ fi
+ done
+ # not all messages only tail the last 10k lines
+ tail -n 10000 /var/log/messages > messages
+}
+
+_curl_data_store(){
+ file=$1
+ url=$2
+ touch $file
+ echo "============================= $url ======================" >> $file
+ curl --silent -u admin:admin -X GET http://$odl_ip_port/$url | python -mjson.tool 2>&1 >> $file
+}
+
+_get_output_karaf(){
+ file=$1
+ touch $file
+ shift
+ echo "============================ KARAF $@ ===================" >> $file
+ sshpass -p karaf ssh -p 8101 -o "StrictHostKeyChecking no" karaf@localhost "$@" 2>&1 >> $file
+
+}
+datastore()
+{
+
+ dump=$tmp_folder/dump-$HOSTNAME.txt
+ operational=$tmp_folder/Operational-Inventory-$HOSTNAME.txt
+ karaf_output=$tmp_folder/Karaf_out-$HOSTNAME.txt
+ odl_ip_port=$(grep ^url= /etc/neutron/plugins/ml2/ml2_conf.ini |cut -d '/' -f3)
+
+ config_urls=( restconf/config/neutron:neutron/networks/ restconf/config/neutron:neutron/subnets/ restconf/config/neutron:neutron/ports/ restconf/config/neutron:neutron/routers/ restconf/config/itm:transport-zones/ restconf/config/itm-state:tunnels_state/ restconf/config/itm-state:external-tunnel-list/ restconf/config/itm-state:dpn-endpoints/ restconf/config/itm-config:vtep-config-schemas/ restconf/config/itm-config:tunnel-monitor-enabled/ restconf/config/itm-config:tunnel-monitor-interval/ restconf/config/interface-service-bindings:service-bindings/ restconf/config/l3vpn:vpn-instances/ restconf/config/ietf-interfaces:interfaces/ restconf/config/l3vpn:vpn-interfaces/ restconf/config/odl-fib:fibEntries restconf/config/neutronvpn:networkMaps restconf/config/neutronvpn:subnetmaps restconf/config/neutronvpn:vpnMaps restconf/config/neutronvpn:neutron-port-data restconf/config/id-manager:id-pools/ restconf/config/elan:elan-instances/ restconf/config/elan:elan-interfaces/ restconf/config/elan:elan-state/ restconf/config/elan:elan-forwarding-tables/ restconf/config/elan:elan-interface-forwarding-entries/ restconf/config/elan:elan-dpn-interfaces/ restconf/config/elan:elan-tag-name-map/ restconf/config/odl-nat:external-networks/ restconf/config/odl-nat:ext-routers/ restconf/config/odl-nat:intext-ip-port-map/ restconf/config/odl-nat:snatint-ip-port-map/ restconf/config/odl-l3vpn:vpn-instance-to-vpn-id/ restconf/config/neutronvpn:neutron-router-dpns/ restconf/operational/itm-config:tunnel-monitor-interval/ restconf/config/itm-config:tunnel-monitor-interval/ restconf/operational/itm-config:tunnel-monitor-params/ restconf/config/itm-config:tunnel-monitor-params/ restconf/config/vpnservice-dhcp:designated-switches-for-external-tunnels/ restconf/config/neutron:neutron/security-groups/ restconf/config/neutron:neutron/security-rules/ restconf/config/network-topology:network-topology/topology/hwvtep:1 restconf/config/network-topology:network-topology/topology/ovsdb:1 )
+
+ for url in ${config_urls[@]};do
+ _curl_data_store $dump $url
+ done
+
+ operational_urls=( restconf/operational/itm-state:tunnels_state/ restconf/operational/odl-interface-meta:bridge-ref-info/ restconf/operational/odl-l3vpn:prefix-to-interface/ restconf/operational/odl-l3vpn:vpn-instance-op-data/ restconf/operational/l3vpn:vpn-interfaces/ restconf/operational/ietf-interfaces:interfaces-state/ restconf/operational/odl-l3vpn:prefix-to-interface/ restconf/operational/l3nexthop:l3nexthop restconf/operational/elan:elan-instances/ restconf/operational/elan:elan-interfaces/ restconf/operational/elan:elan-state/ restconf/operational/elan:elan-forwarding-tables/ restconf/operational/elan:elan-interface-forwarding-entries/ restconf/operational/elan:elan-dpn-interfaces/ restconf/operational/elan:elan-tag-name-map/ restconf/operational/odl-nat:napt-switches/ restconf/operational/odl-nat:intext-ip-map/ restconf/operational/odl-nat:external-ips-counter/ restconf/operational/neutronvpn:neutron-router-dpns/ restconf/operational/odl-l3vpn:subnet-op-data/ restconf/operational/odl-l3vpn:port-op-data/ restconf/operational/odl-l3vpn:vpn-to-extraroute restconf/operational/neutronvpn:neutron-vpn-portip-port-data/ restconf/operational/odl-fib:label-route-map/ restconf/operational/itm-config:tunnel-monitor-interval/ restconf/operational/itm-config:tunnel-monitor-params/ restconf/operational/odl-interface-meta:if-indexes-interface-map/ restconf/operational/network-topology:network-topology/topology/hwvtep:1 restconf/operational/network-topology:network-topology/topology/ovsdb:1 restconf/operational/opendaylight-inventory:nodes/ restconf/operational/opendaylight-inventory:nodes/ )
+
+ for url in ${operational_urls[@]};do
+ _curl_data_store $operational $url
+ done
+
+
+ karaf_commands=( "display-bgp-config --debug" "bundle:list -s" fib-show "show-bgp --cmd 'ip bgp summary'" "show-bgp --cmd 'ip bgp neighbor'" "show-bgp --cmd 'ip bgp vpnv4 all summary'")
+ for ((i = 0; i < ${#karaf_commands[@]}; i++));do
+ _get_output_karaf $karaf_output ${karaf_commands[$i]}
+ done
+ # TODO check this one:
+ # curl --silent -u admin:admin -H "Content-Type: application/json" --data "{"input" : { } }" -X POST http://$OS_MIP:8181/restconf/operations/neutronvpn:getL3VPN | python -mjson.tool >>$file1
+}
+
+# check if running on karaf node
+if netstat -atnp |grep 8101 -q;then
+ datastore
+fi
+flows
+node
+
+pushd /tmp
+rm -rf log_output.tar.gz
+#tar -czvf dump_output-$HOSTNAME-`date +"%d-%m-%H-%M"`.tar.gz $tmp_folder
+tar -czvf log_output.tar.gz $tmp_folder 2>&1
+
diff --git a/sdnvpn/test/functest/run_tests.py b/sdnvpn/test/functest/run_tests.py
index a861a47..1bffe67 100644
--- a/sdnvpn/test/functest/run_tests.py
+++ b/sdnvpn/test/functest/run_tests.py
@@ -19,7 +19,7 @@ import yaml
import functest.utils.functest_utils as ft_utils
from sdnvpn.lib import config as sdnvpn_config
-
+from sdnvpn.lib.gather_logs import gather_logs
logger = logging.getLogger('sdnvpn-run-tests')
@@ -93,6 +93,12 @@ def main(report=False):
push_results(
test_name_db, start_time, end_time, status, details)
+ try:
+ gather_logs('overall')
+ except Exception as ex:
+ logger.error(('Something went wrong in the Log gathering.'
+ 'Ex: Trace: %s')
+ % ex, traceback.format_exc())
if overall_status == "FAIL":
sys.exit(-1)