diff options
Diffstat (limited to 'tools/collectors')
| -rw-r--r-- | tools/collectors/collector/collector.py | 37 | ||||
| -rwxr-xr-x | tools/collectors/sysmetrics/__init__.py | 5 | ||||
| -rw-r--r-- | tools/collectors/sysmetrics/linuxmetrics.py | 79 | ||||
| -rw-r--r-- | tools/collectors/sysmetrics/pidstat.py | 140 |
4 files changed, 171 insertions, 90 deletions
diff --git a/tools/collectors/collector/collector.py b/tools/collectors/collector/collector.py index 27a07202..998c1f6f 100644 --- a/tools/collectors/collector/collector.py +++ b/tools/collectors/collector/collector.py @@ -15,24 +15,49 @@ """Abstract "system metrics logger" model. """ -CMD_PREFIX = 'metricscmd : ' - class ICollector(object): """This is an abstract class for system metrics loggers. """ - def log_mem_stats(self): - """Log memory statistics. + def start(self): + """Starts data collection. This method must be non-blocking. + It means, that collector must be executed as a background process. Where implemented, this function should raise an exception on failure. """ raise NotImplementedError('Please call an implementation.') - def log_cpu_stats(self): - """Log cpu statistics. + def stop(self): + """Stops data collection. Where implemented, this function should raise an exception on failure. """ raise NotImplementedError('Please call an implementation.') + + def get_results(self): + """Returns collected results. + + Where implemented, this function should raise an exception on + failure. + """ + raise NotImplementedError('Please call an implementation.') + + def print_results(self): + """Logs collected results. + + Where implemented, this function should raise an exception on + failure. + """ + raise NotImplementedError('Please call an implementation.') + + def __enter__(self): + """Starts up collection of statistics + """ + self.start() + + def __exit__(self, type_, value, traceback): + """Stops collection of statistics + """ + self.stop() diff --git a/tools/collectors/sysmetrics/__init__.py b/tools/collectors/sysmetrics/__init__.py index 9ad1bf29..e3e07fd8 100755 --- a/tools/collectors/sysmetrics/__init__.py +++ b/tools/collectors/sysmetrics/__init__.py @@ -11,8 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Implementation of linux-metrics system metrics logger. -""" - -from tools.collectors.sysmetrics.linuxmetrics import * diff --git a/tools/collectors/sysmetrics/linuxmetrics.py b/tools/collectors/sysmetrics/linuxmetrics.py deleted file mode 100644 index fdf30696..00000000 --- a/tools/collectors/sysmetrics/linuxmetrics.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""linux-metrics system statistics model. - -Provides linux-metrics system statistics generic "helper" functions. - -This requires the following setting in your config: - -* SYSMETRICS_LINUX_METRICS_CPU_SAMPLES_INTERVAL - Number of seconds in between samples to take for CPU percentages - -If this doesn't exist, the application will raise an exception -(EAFP). -""" - - -import logging -import os -from conf import settings -from tools.collectors.collector import collector -from linux_metrics import cpu_stat, mem_stat - -_ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) - -class LinuxMetrics(collector.ICollector): - """A logger based on the linux-metrics module. - - Currently it supports the logging of memory and CPU statistics - """ - def __init__(self): - self._logger = logging.getLogger(__name__) - self._num_samples = settings.getValue( - 'SYSMETRICS_LINUX_METRICS_CPU_SAMPLES_INTERVAL') - self._mem_stats = [] - self._cpu_stats = [] - - def log_mem_stats(self): - """See ICollector for descripion - """ - self._mem_stats = mem_stat.mem_stats() - # pylint: disable=unbalanced-tuple-unpacking - mem_active, mem_total, mem_cached, mem_free, swap_total, swap_free = \ - self._mem_stats - self._logger.info('%s mem_active: %s, mem_total: %s, mem_cached: %s, ' - 'mem_free: %s, swap_total: %s, swap_free: %s', - collector.CMD_PREFIX, - mem_active, mem_total, mem_cached, mem_free, - swap_total, swap_free) - return self._mem_stats - - def log_cpu_stats(self): - """See ICollector for descripion - """ - self._cpu_stats = cpu_stat.cpu_percents(self._num_samples) - self._logger.info('%s user: %.2f%%, nice: %.2f%%, system: %.2f%%, ' - 'idle: %.2f%%, iowait: %.2f%%, irq: %.2f%%, ' - 'softirq: %.2f%%', - collector.CMD_PREFIX, - self._cpu_stats['user'], - self._cpu_stats['nice'], - self._cpu_stats['system'], - self._cpu_stats['idle'], - self._cpu_stats['iowait'], - self._cpu_stats['irq'], - self._cpu_stats['softirq']) - return self._cpu_stats - diff --git a/tools/collectors/sysmetrics/pidstat.py b/tools/collectors/sysmetrics/pidstat.py new file mode 100644 index 00000000..608a0d6c --- /dev/null +++ b/tools/collectors/sysmetrics/pidstat.py @@ -0,0 +1,140 @@ +# Copyright 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""module for statistics collection by pidstat + +Provides system statistics collected between calls of start() and stop() +by command line tool pidstat (part of sysstat package) + +This requires the following setting in your config: + +* PIDSTAT_MONITOR = ['ovs-vswitchd', 'ovsdb-server', 'kvm'] + processes to be monitorred by pidstat + +* PIDSTAT_OPTIONS = '-dur' + options which will be passed to pidstat, i.e. what + statistics should be collected by pidstat + +* LOG_FILE_PIDSTAT = 'pidstat.log' + log file for pidstat; it defines suffix, which will be added + to testcase name. Pidstat detailed statistics will be stored separately + for every testcase. + +If this doesn't exist, the application will raise an exception +(EAFP). +""" + +import os +import logging +import subprocess +import time +from collections import OrderedDict +from tools import tasks +from tools import systeminfo +from conf import settings +from tools.collectors.collector import collector + +_ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) + +class Pidstat(collector.ICollector): + """A logger of system statistics based on pidstat + + It collects statistics based on configuration + """ + _logger = logging.getLogger(__name__) + + def __init__(self, results_dir, test_name): + """ + Initialize collection of statistics + """ + self._log = os.path.join(results_dir, + settings.getValue('LOG_FILE_PIDSTAT') + + '_' + test_name + '.log') + self._results = OrderedDict() + self._pid = 0 + + def start(self): + """ + Starts collection of statistics by pidstat and stores them + into the file in directory with test results + """ + monitor = settings.getValue('PIDSTAT_MONITOR') + self._logger.info('Statistics are requested for: ' + ', '.join(monitor)) + pids = systeminfo.get_pids(monitor) + if pids: + with open(self._log, 'w') as logfile: + cmd = ['sudo', 'pidstat', settings.getValue('PIDSTAT_OPTIONS'), + '-p', ','.join(pids), + str(settings.getValue('PIDSTAT_SAMPLE_INTERVAL'))] + self._logger.debug('%s', ' '.join(cmd)) + self._pid = subprocess.Popen(cmd, stdout=logfile, bufsize=0).pid + + def stop(self): + """ + Stops collection of statistics by pidstat and stores statistic summary + for each monitored process into self._results dictionary + """ + if self._pid: + self._pid = 0 + # in python3.4 it's not possible to send signal through pid of sudo + # process, so all pidstat processes are interupted instead + # as a workaround + tasks.run_task(['sudo', 'pkill', '--signal', '2', 'pidstat'], + self._logger) + + self._logger.info( + 'Pidstat log available at %s', self._log) + + # let's give pidstat some time to write down average summary + time.sleep(2) + + # parse average values from log file and store them to _results dict + self._results = OrderedDict() + logfile = open(self._log, 'r') + with logfile: + line = logfile.readline() + while line: + line = line.strip() + # process only lines with summary + if line[0:7] == 'Average': + if line[-7:] == 'Command': + # store header fields if detected + tmp_header = line[8:].split() + else: + # combine stored header fields with actual values + tmp_res = OrderedDict(zip(tmp_header, + line[8:].split())) + # use process's name and its pid as unique key + key = tmp_res.pop('Command') + '_' + tmp_res['PID'] + # store values for given command into results dict + if key in self._results: + self._results[key].update(tmp_res) + else: + self._results[key] = tmp_res + + line = logfile.readline() + + def get_results(self): + """Returns collected statistics. + """ + return self._results + + def print_results(self): + """Logs collected statistics. + """ + for process in self._results: + logging.info("Process: " + '_'.join(process.split('_')[:-1])) + for(key, value) in self._results[process].items(): + logging.info(" Statistic: " + str(key) + + ", Value: " + str(value)) |