summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorahothan <ahothan@cisco.com>2017-08-22 17:41:16 -0700
committerahothan <ahothan@cisco.com>2017-08-22 17:41:57 -0700
commit2d55474511a5057015e77547c326120c1649c0b7 (patch)
treee5f1db615655d73c9778d0499d7a312e8de2cc3e
parent6c4e55c53c65b7e63c17c85367b4813443c5d942 (diff)
NFVBENCH-6 Add support for sending logs to fluentd with fluentd client library
Change-Id: I1bc01b26f9e43f78c169b5fcd26247debcfe31bf Signed-off-by: ahothan <ahothan@cisco.com>
-rw-r--r--nfvbench/cfg.default.yaml14
-rw-r--r--nfvbench/fluentd.py47
-rw-r--r--nfvbench/log.py15
-rw-r--r--nfvbench/nfvbench.py16
-rw-r--r--requirements.txt1
-rw-r--r--test/test_nfvbench.py18
6 files changed, 107 insertions, 4 deletions
diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml
index dba7670..a082a8f 100644
--- a/nfvbench/cfg.default.yaml
+++ b/nfvbench/cfg.default.yaml
@@ -369,6 +369,20 @@ debug: false
# Defaults to disabled
log_file:
+# When enabled, all logs will be sent to a fluentd server at the requested IP and port
+# The fluentd "tag" and "label" fields for every message will be set to "nfvbench"
+fluentd:
+ # by default (logging_tag is empty) nfvbench log messages are not sent to fluentd
+ # to enable logging to fluents, specify a valid fluentd tag name to be used for the
+ # log records
+ logging_tag:
+
+ # IP address of the server, defaults to loopback
+ ip: 127.0.0.1
+
+ # port # to use, by default, use the default fluentd forward port
+ port: 24224
+
# Module and class name of factory which will be used to provide classes dynamically for other components.
factory_module: 'nfvbench.factory'
factory_class: 'BasicFactory'
diff --git a/nfvbench/fluentd.py b/nfvbench/fluentd.py
new file mode 100644
index 0000000..683d4ce
--- /dev/null
+++ b/nfvbench/fluentd.py
@@ -0,0 +1,47 @@
+# Copyright 2017 Cisco Systems, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from datetime import datetime
+from fluent import sender
+import logging
+
+class FluentLogHandler(logging.Handler):
+ '''This is a minimalist log handler for use with Fluentd
+
+ Needs to be attached to a logger using the addHandler method.
+ It only picks up from every record:
+ - the formatted message (no timestamp and no level)
+ - the level name
+ - the runlogdate (to tie multiple run-related logs together)
+ The timestamp is retrieved by the fluentd library.
+ '''
+ def __init__(self, tag, fluentd_ip='127.0.0.1', fluentd_port=24224):
+ logging.Handler.__init__(self)
+ self.tag = tag
+ self.formatter = logging.Formatter('%(message)s')
+ self.sender = sender.FluentSender(self.tag, port=fluentd_port)
+ self.start_new_run()
+
+ def start_new_run(self):
+ '''Delimitate a new run in the stream of records with a new timestamp
+ '''
+ self.runlogdate = str(datetime.now())
+
+ def emit(self, record):
+ data = {
+ "runlogdate": self.runlogdate,
+ "loglevel": record.levelname,
+ "message": self.formatter.format(record)
+ }
+ self.sender.emit(None, data)
diff --git a/nfvbench/log.py b/nfvbench/log.py
index f308171..674ddf8 100644
--- a/nfvbench/log.py
+++ b/nfvbench/log.py
@@ -16,15 +16,22 @@ import logging
_product_name = 'nfvbench'
-def setup():
+def setup(mute_stdout=False):
# logging.basicConfig()
- formatter_str = '%(asctime)s %(levelname)s %(message)s'
- handler = logging.StreamHandler()
- handler.setFormatter(logging.Formatter(formatter_str))
+ if mute_stdout:
+ handler = logging.NullHandler()
+ else:
+ formatter_str = '%(asctime)s %(levelname)s %(message)s'
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter(formatter_str))
# Add handler to logger
logger = logging.getLogger(_product_name)
logger.addHandler(handler)
+ # disable unnecessary information capture
+ logging.logThreads = 0
+ logging.logProcesses = 0
+ logging._srcfile = None
def add_file_logger(logfile):
if logfile:
diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py
index d3f7f02..dccd63c 100644
--- a/nfvbench/nfvbench.py
+++ b/nfvbench/nfvbench.py
@@ -26,6 +26,7 @@ import copy
import credentials
import datetime
from factory import BasicFactory
+from fluentd import FluentLogHandler
import importlib
import json
import log
@@ -42,6 +43,7 @@ import traceback
from traffic_client import TrafficGeneratorFactory
import utils
+fluent_logger = None
class NFVBench(object):
"""Main class of NFV benchmarking tool."""
@@ -78,6 +80,10 @@ class NFVBench(object):
status = NFVBench.STATUS_OK
result = None
message = ''
+ if fluent_logger:
+ # take a snapshot of the current time for this new run
+ # so that all subsequent logs can relate to this run
+ fluent_logger.start_new_run()
try:
self.update_config(opts)
self.setup()
@@ -411,6 +417,7 @@ def check_physnet(name, netattrs):
.format(n=name))
def main():
+ global fluent_logger
try:
log.setup()
# load default config file
@@ -427,6 +434,15 @@ def main():
config = config_plugin.get_config()
openstack_spec = config_plugin.get_openstack_spec()
+ # setup the fluent logger as soon as possible right after the config plugin is called
+ if config.fluentd.logging_tag:
+ fluent_logger = FluentLogHandler(config.fluentd.logging_tag,
+ fluentd_ip=config.fluentd.ip,
+ fluentd_port=config.fluentd.port)
+ LOG.addHandler(fluent_logger)
+ else:
+ fluent_logger = None
+
opts, unknown_opts = parse_opts_from_cli()
log.set_level(debug=opts.debug)
diff --git a/requirements.txt b/requirements.txt
index 2bb548d..5cc6989 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,3 +24,4 @@ flask_socketio>=2.8.3
backports.ssl-match-hostname==3.5.0.1 # via websocket-client
socketIO-client==0.7.2
websocket-client==0.40.0 # via socketio-client
+fluent-logger>=0.5.2 \ No newline at end of file
diff --git a/test/test_nfvbench.py b/test/test_nfvbench.py
index a70e71a..ad78a9e 100644
--- a/test/test_nfvbench.py
+++ b/test/test_nfvbench.py
@@ -15,9 +15,12 @@
#
from attrdict import AttrDict
+import logging
from nfvbench.config import get_err_config
from nfvbench.connection import SSH
from nfvbench.credentials import Credentials
+from nfvbench.fluentd import FluentLogHandler
+import nfvbench.log
from nfvbench.network import Interface
from nfvbench.network import Network
from nfvbench.specs import Encaps
@@ -632,6 +635,9 @@ def test_ndr_pdr_search(traffic_client):
# Other tests
# =========================================================================
+def setup_module(module):
+ nfvbench.log.setup(mute_stdout=True)
+
def test_no_credentials():
cred = Credentials('/completely/wrong/path/openrc', None, False)
if cred.rc_auth_url:
@@ -656,3 +662,15 @@ def test_config():
assert(get_err_config({2: 100}, refcfg) == {2: 100})
# both correctly fail and invalid value type
assert(get_err_config({2: 100, 5: 10}, refcfg) == {2: 100, 5: 10})
+
+def test_fluentd():
+ logger = logging.getLogger('fluent-logger')
+ handler = FluentLogHandler('nfvbench', fluentd_port=7081)
+ logger.addHandler(handler)
+ logger.setLevel(logging.INFO)
+ logger.info('test')
+ logger.warning('test %d', 100)
+ try:
+ raise Exception("test")
+ except Exception:
+ logger.exception("got exception")