From 2d55474511a5057015e77547c326120c1649c0b7 Mon Sep 17 00:00:00 2001 From: ahothan Date: Tue, 22 Aug 2017 17:41:16 -0700 Subject: NFVBENCH-6 Add support for sending logs to fluentd with fluentd client library Change-Id: I1bc01b26f9e43f78c169b5fcd26247debcfe31bf Signed-off-by: ahothan --- nfvbench/cfg.default.yaml | 14 ++++++++++++++ nfvbench/fluentd.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ nfvbench/log.py | 15 +++++++++++---- nfvbench/nfvbench.py | 16 ++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 nfvbench/fluentd.py (limited to 'nfvbench') diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml index dba7670..a082a8f 100644 --- a/nfvbench/cfg.default.yaml +++ b/nfvbench/cfg.default.yaml @@ -369,6 +369,20 @@ debug: false # Defaults to disabled log_file: +# When enabled, all logs will be sent to a fluentd server at the requested IP and port +# The fluentd "tag" and "label" fields for every message will be set to "nfvbench" +fluentd: + # by default (logging_tag is empty) nfvbench log messages are not sent to fluentd + # to enable logging to fluents, specify a valid fluentd tag name to be used for the + # log records + logging_tag: + + # IP address of the server, defaults to loopback + ip: 127.0.0.1 + + # port # to use, by default, use the default fluentd forward port + port: 24224 + # Module and class name of factory which will be used to provide classes dynamically for other components. factory_module: 'nfvbench.factory' factory_class: 'BasicFactory' diff --git a/nfvbench/fluentd.py b/nfvbench/fluentd.py new file mode 100644 index 0000000..683d4ce --- /dev/null +++ b/nfvbench/fluentd.py @@ -0,0 +1,47 @@ +# Copyright 2017 Cisco Systems, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from datetime import datetime +from fluent import sender +import logging + +class FluentLogHandler(logging.Handler): + '''This is a minimalist log handler for use with Fluentd + + Needs to be attached to a logger using the addHandler method. + It only picks up from every record: + - the formatted message (no timestamp and no level) + - the level name + - the runlogdate (to tie multiple run-related logs together) + The timestamp is retrieved by the fluentd library. + ''' + def __init__(self, tag, fluentd_ip='127.0.0.1', fluentd_port=24224): + logging.Handler.__init__(self) + self.tag = tag + self.formatter = logging.Formatter('%(message)s') + self.sender = sender.FluentSender(self.tag, port=fluentd_port) + self.start_new_run() + + def start_new_run(self): + '''Delimitate a new run in the stream of records with a new timestamp + ''' + self.runlogdate = str(datetime.now()) + + def emit(self, record): + data = { + "runlogdate": self.runlogdate, + "loglevel": record.levelname, + "message": self.formatter.format(record) + } + self.sender.emit(None, data) diff --git a/nfvbench/log.py b/nfvbench/log.py index f308171..674ddf8 100644 --- a/nfvbench/log.py +++ b/nfvbench/log.py @@ -16,15 +16,22 @@ import logging _product_name = 'nfvbench' -def setup(): +def setup(mute_stdout=False): # logging.basicConfig() - formatter_str = '%(asctime)s %(levelname)s %(message)s' - handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter(formatter_str)) + if mute_stdout: + handler = logging.NullHandler() + else: + formatter_str = '%(asctime)s %(levelname)s %(message)s' + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(formatter_str)) # Add handler to logger logger = logging.getLogger(_product_name) logger.addHandler(handler) + # disable unnecessary information capture + logging.logThreads = 0 + logging.logProcesses = 0 + logging._srcfile = None def add_file_logger(logfile): if logfile: diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py index d3f7f02..dccd63c 100644 --- a/nfvbench/nfvbench.py +++ b/nfvbench/nfvbench.py @@ -26,6 +26,7 @@ import copy import credentials import datetime from factory import BasicFactory +from fluentd import FluentLogHandler import importlib import json import log @@ -42,6 +43,7 @@ import traceback from traffic_client import TrafficGeneratorFactory import utils +fluent_logger = None class NFVBench(object): """Main class of NFV benchmarking tool.""" @@ -78,6 +80,10 @@ class NFVBench(object): status = NFVBench.STATUS_OK result = None message = '' + if fluent_logger: + # take a snapshot of the current time for this new run + # so that all subsequent logs can relate to this run + fluent_logger.start_new_run() try: self.update_config(opts) self.setup() @@ -411,6 +417,7 @@ def check_physnet(name, netattrs): .format(n=name)) def main(): + global fluent_logger try: log.setup() # load default config file @@ -427,6 +434,15 @@ def main(): config = config_plugin.get_config() openstack_spec = config_plugin.get_openstack_spec() + # setup the fluent logger as soon as possible right after the config plugin is called + if config.fluentd.logging_tag: + fluent_logger = FluentLogHandler(config.fluentd.logging_tag, + fluentd_ip=config.fluentd.ip, + fluentd_port=config.fluentd.port) + LOG.addHandler(fluent_logger) + else: + fluent_logger = None + opts, unknown_opts = parse_opts_from_cli() log.set_level(debug=opts.debug) -- cgit 1.2.3-korg