aboutsummaryrefslogtreecommitdiffstats
path: root/puppet-barometer
ModeNameSize
-rw-r--r--README.md2600logstatsplain
d---------lib / puppet33logstatsplain
d---------manifests354logstatsplain
-rw-r--r--metadata.json1247logstatsplain
d---------releasenotes / source33logstatsplain
-rw-r--r--setup.py781logstatsplain
d---------spec111logstatsplain
d---------templates599logstatsplain
-rw-r--r--test-requirements.txt134logstatsplain
d---------tests35logstatsplain
-rw-r--r--tox.ini234logstatsplain
.nd { color: #a6e22e } /* Name.Decorator */ .highlight .ni { color: #f8f8f2 } /* Name.Entity */ .highlight .ne { color: #a6e22e } /* Name.Exception */ .highlight .nf { color: #a6e22e } /* Name.Function */ .highlight .nl { color: #f8f8f2 } /* Name.Label */ .highlight .nn { color: #f8f8f2 } /* Name.Namespace */ .highlight .nx { color: #a6e22e } /* Name.Other */ .highlight .py { color: #f8f8f2 } /* Name.Property */ .highlight .nt { color: #f92672 } /* Name.Tag */ .highlight .nv { color: #f8f8f2 } /* Name.Variable */ .highlight .ow { color: #f92672 } /* Operator.Word */ .highlight .w { color: #f8f8f2 } /* Text.Whitespace */ .highlight .mb { color: #ae81ff } /* Literal.Number.Bin */ .highlight .mf { color: #ae81ff } /* Literal.Number.Float */ .highlight .mh { color: #ae81ff } /* Literal.Number.Hex */ .highlight .mi { color: #ae81ff } /* Literal.Number.Integer */ .highlight .mo { color: #ae81ff } /* Literal.Number.Oct */ .highlight .sa { color: #e6db74 } /* Literal.String.Affix */ .highlight .sb { color: #e6db74 } /* Literal.String.Backtick */ .highlight .sc { color: #e6db74 } /* Literal.String.Char */ .highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */ .highlight .sd { color: #e6db74 } /* Literal.String.Doc */ .highlight .s2 { color: #e6db74 } /* Literal.String.Double */ .highlight .se { color: #ae81ff } /* Literal.String.Escape */ .highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */ .highlight .si { color: #e6db74 } /* Literal.String.Interpol */ .highlight .sx { color: #e6db74 } /* Literal.String.Other */ .highlight .sr { color: #e6db74 } /* Literal.String.Regex */ .highlight .s1 { color: #e6db74 } /* Literal.String.Single */ .highlight .ss { color: #e6db74 } /* Literal.String.Symbol */ .highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #a6e22e } /* Name.Function.Magic */ .highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */ .highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */ .highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */ .highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */ .highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */ } @media (prefers-color-scheme: light) { .highlight .hll { background-color: #ffffcc } .highlight .c { color: #888888 } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ .highlight .k { color: #008800; font-weight: bold } /* Keyword */ .highlight .ch { color: #888888 } /* Comment.Hashbang */ .highlight .cm { color: #888888 } /* Comment.Multiline */ .highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */ .highlight .cpf { color: #888888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ }
# Copyright 2015-2017 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Automation of hugepages management
"""

import os
import re
import subprocess
import logging
import locale
import math

from tools import tasks
from conf import settings

_LOGGER = logging.getLogger(__name__)
_ALLOCATED_HUGEPAGES = False
#
# hugepage management
#


def get_hugepage_size():
    """Return the size of the configured hugepages
    """
    hugepage_size_re = re.compile(r'^Hugepagesize:\s+(?P<size_hp>\d+)\s+kB',
                                  re.IGNORECASE)
    with open('/proc/meminfo', 'r') as result_file:
        data = result_file.readlines()
        for line in data:
            match = hugepage_size_re.search(line)
            if match:
                _LOGGER.info('Hugepages size: %s kb', match.group('size_hp'))
                return int(match.group('size_hp'))
        _LOGGER.error('Could not parse for hugepage size')
        return 0


def allocate_hugepages():
    """Allocate hugepages on the fly
    """
    hp_size = get_hugepage_size()
    if hp_size > 0:
        nr_hp = int(math.ceil(settings.getValue('HUGEPAGE_RAM_ALLOCATION')/hp_size))
        _LOGGER.info('Will allocate %s hugepages.', nr_hp)

        nr_hugepages = 'vm.nr_hugepages=' + str(nr_hp)
        try:
            tasks.run_task(['sudo', 'sysctl', nr_hugepages],
                           _LOGGER, 'Trying to allocate hugepages..', True)
        except subprocess.CalledProcessError:
            _LOGGER.error('Unable to allocate hugepages.')
            return False
        # pylint: disable=global-statement
        global _ALLOCATED_HUGEPAGES
        _ALLOCATED_HUGEPAGES = True
        return True

    else:
        _LOGGER.error('Division by 0 will be supported in next release')
        return False

def deallocate_hugepages():
    """De-allocate hugepages that were allocated on the fly
    """
    # pylint: disable=global-statement
    global _ALLOCATED_HUGEPAGES
    if _ALLOCATED_HUGEPAGES:
        nr_hugepages = 'vm.nr_hugepages= 0'
        try:
            tasks.run_task(['sudo', 'sysctl', nr_hugepages],
                           _LOGGER, 'Trying to de-allocate hugepages..', True)
        except subprocess.CalledProcessError:
            _LOGGER.error('Unable to de-allocate hugepages.')
            return False
        _ALLOCATED_HUGEPAGES = False
    return True


def get_free_hugepages(socket=None):
    """Get the free hugepage totals on the system.

    :param socket: optional socket param to get free hugepages on a socket. To
                   be passed a string.
    :returns: hugepage amount as int
    """
    hugepage_free_re = re.compile(r'HugePages_Free:\s+(?P<free_hp>\d+)$')
    if socket:
        if os.path.exists(
                '/sys/devices/system/node/node{}/meminfo'.format(socket)):
            meminfo_path = '/sys/devices/system/node/node{}/meminfo'.format(
                socket)
        else:
            _LOGGER.info('No hugepage info found for socket %s', socket)
            return 0
    else:
        meminfo_path = '/proc/meminfo'

    with open(meminfo_path, 'r') as result_file:
        data = result_file.readlines()
        for line in data:
            match = hugepage_free_re.search(line)
            if match:
                _LOGGER.info('Hugepages free: %s %s', match.group('free_hp'),
                             'on socket {}'.format(socket) if socket else '')
                return int(match.group('free_hp'))
        _LOGGER.info('Could not parse for hugepage size')
        return 0


def is_hugepage_available():
    """Check if hugepages are configured/available on the system.
    """
    hugepage_size_re = re.compile(r'^Hugepagesize:\s+(?P<size_hp>\d+)\s+kB',
                                  re.IGNORECASE)

    # read in meminfo
    with open('/proc/meminfo') as mem_file:
        mem_info = mem_file.readlines()

    # see if the hugepage size is the recommended value
    for line in mem_info:
        match_size = hugepage_size_re.match(line)
        if match_size:
            if match_size.group('size_hp') != '1048576':
                _LOGGER.info(
                    '%s%s%s kB',
                    'Hugepages not configured for recommend 1GB size. ',
                    'Currently set at ', match_size.group('size_hp'))
    num_huge = get_free_hugepages()
    if num_huge == 0:
        _LOGGER.info('No free hugepages.')
        if not allocate_hugepages():
            return False
    else:
        _LOGGER.info('Found \'%s\' free hugepage(s).', num_huge)
    return True


def is_hugepage_mounted():
    """Check if hugepages are mounted.
    """
    output = subprocess.check_output(['mount'], shell=True)
    my_encoding = locale.getdefaultlocale()[1]
    for line in output.decode(my_encoding).split('\n'):
        if 'hugetlbfs' in line:
            return True

    return False


def mount_hugepages():
    """Ensure hugepages are mounted. Raises RuntimeError if no configured
    hugepages are available.
    """
    if not is_hugepage_available():
        raise RuntimeError('No Hugepages configured.')

    if is_hugepage_mounted():
        return

    if not os.path.exists(settings.getValue('HUGEPAGE_DIR')):
        tasks.run_task(['sudo', 'mkdir', settings.getValue('HUGEPAGE_DIR')], _LOGGER,
                       'Creating directory ' + settings.getValue('HUGEPAGE_DIR'), True)
    try:
        tasks.run_task(['sudo', 'mount', '-t', 'hugetlbfs', 'nodev',
                        settings.getValue('HUGEPAGE_DIR')],
                       _LOGGER, 'Mounting hugepages...', True)
    except subprocess.CalledProcessError:
        _LOGGER.error('Unable to mount hugepages.')


def umount_hugepages():
    """Ensure hugepages are unmounted.
    """
    if not is_hugepage_mounted():
        return

    try:
        tasks.run_task(['sudo', 'umount', settings.getValue('HUGEPAGE_DIR')],
                       _LOGGER, 'Unmounting hugepages...', True)
    except subprocess.CalledProcessError:
        _LOGGER.error('Unable to umount hugepages.')

    if not deallocate_hugepages():
        _LOGGER.error('Unable to deallocate previously allocated hugepages.')