From fb0f275cd373aec28a8a970ced0f1d20ca8abbfc Mon Sep 17 00:00:00 2001 From: Michael Chapman Date: Mon, 9 May 2016 17:38:54 +1000 Subject: Add pinning support Add support for CPU pinning of libvirt guests and also restricting host processes to selected CPUs via isolcpus. Hugepage support is added using the same mechanism as isolcpus, along with a perf scenario where all 3 performance options are enabled. Deploy options are now parsed in python JIRA: APEX-127 JIRA: APEX-105 JIRA: APEX-106 Change-Id: I438e80fb88e596cc017595d43bc1efda1001325c opnfv-tht-pr: 8 Signed-off-by: Michael Chapman --- build/build_perf_image.sh | 38 ++++++++ build/overcloud-full.sh | 8 ++ build/set_perf_images.sh | 31 ++++++ build/setkernelparam.sh | 27 ++++++ build/undercloud.sh | 8 +- ci/deploy.sh | 69 +++++++------ config/deploy/os-nosdn-performance-ha.yaml | 23 +++++ lib/python/apex-python-utils.py | 11 ++- lib/python/apex/__init__.py | 1 + lib/python/apex/deploy_env.py | 149 +++++++++++++++++++++++++++++ 10 files changed, 325 insertions(+), 40 deletions(-) create mode 100644 build/build_perf_image.sh create mode 100644 build/set_perf_images.sh create mode 100644 build/setkernelparam.sh create mode 100644 config/deploy/os-nosdn-performance-ha.yaml create mode 100644 lib/python/apex/deploy_env.py diff --git a/build/build_perf_image.sh b/build/build_perf_image.sh new file mode 100644 index 00000000..0a3e3d0f --- /dev/null +++ b/build/build_perf_image.sh @@ -0,0 +1,38 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 Red Hat Inc. +# Michael Chapman +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +ROLE=$1 +shift +CATEGORY=$1 +shift +KEY=$1 +shift +VALUE=$1 +shift + +IMAGE=$ROLE-overcloud-full.qcow2 + +# Create image copy for this role +if [ ! -f $IMAGE ] ; then + cp overcloud-full.qcow2 $IMAGE +fi + +if [ "$CATEGORY" == "nova" ]; then + if [ "$KEY" == "libvirtpin" ]; then + sudo sed -i "s/#LibvirtCPUPinSet:.*/LibvirtCPUPinSet: '${VALUE}'/" /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml + fi +fi + +if [ "$CATEGORY" == "kernel" ]; then + LIBGUESTFS_BACKEND=direct virt-customize \ + --run-command "bash -x /root/setkernelparam.sh $KEY $VALUE" \ + -a $IMAGE +fi + diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh index 065201f8..d0ee3f01 100755 --- a/build/overcloud-full.sh +++ b/build/overcloud-full.sh @@ -29,5 +29,13 @@ LIBGUESTFS_BACKEND=direct virt-customize \ --run-command "echo 'nf_conntrack_proto_sctp' > /etc/modules-load.d/nf_conntrack_proto_sctp.conf" \ -a overcloud-full_build.qcow2 +################################### +##### Add CPU pinning script ##### +################################### + +LIBGUESTFS_BACKEND=direct virt-customize \ + --upload ../setkernelparam.sh:/root \ + -a overcloud-full_build.qcow2 + mv -f overcloud-full_build.qcow2 overcloud-full.qcow2 popd > /dev/null diff --git a/build/set_perf_images.sh b/build/set_perf_images.sh new file mode 100644 index 00000000..ea31c7fa --- /dev/null +++ b/build/set_perf_images.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2016 Red Hat Inc. +# Michael Chapman +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +for ROLE in $@; do + if [ -f $ROLE-overcloud-full.qcow2 ]; then + echo "Uploading $ROLE-overcloud-full.qcow2 " + KERNEL=$(glance image-show overcloud-full | grep 'kernel_id' | cut -d '|' -f 3 | xargs) + RAMDISK=$(glance image-show overcloud-full | grep 'ramdisk_id' | cut -d '|' -f 3 | xargs) + glance image-create --name $ROLE-overcloud-full --disk-format qcow2 --file $ROLE-overcloud-full.qcow2 --container-format bare --property ramdisk_id=$RAMDISK --property kernel_id=$KERNEL + fi + + if [ "$ROLE" == "Controller" ]; then + sed -i "s/overcloud-full/Controller-overcloud-full" opnfv-environment.yaml + fi + + if [ "$ROLE" == "Compute" ]; then + sudo sed -i "s/NovaImage: overcloud-full/Compute-overcloud-full/" /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml + fi + + if [ "$ROLE" == "BlockStorage" ]; then + sudo sed -i "s/BlockStorageImage: overcloud-full/BlockStorage-overcloud-full/" /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml + fi +done diff --git a/build/setkernelparam.sh b/build/setkernelparam.sh new file mode 100644 index 00000000..b6986d6c --- /dev/null +++ b/build/setkernelparam.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2016 Red Hat Inc. +# Michael Chapman +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +GRUBCONF='/boot/grub2/grub.conf' + +if [ "$1" == "" ]; then + echo "No kernel parameter name provided, not modifying grub.conf" + exit 1 +fi + +if [ "$2" == "" ]; then + echo "No kernel parameter value provided, not modifying grub.conf" + exit 1 +fi + +echo "Setting $1=$2 in $GRUBCONF" +echo "GRUB_CMDLINE_LINUX=\"\$GRUB_CMDLINE_LINUX $1=$2\"" >> /etc/default/grub +grub2-mkconfig > $GRUBCONF +exit 0 diff --git a/build/undercloud.sh b/build/undercloud.sh index ed4f2b34..04c2667f 100755 --- a/build/undercloud.sh +++ b/build/undercloud.sh @@ -43,7 +43,7 @@ if [ "$PR_NUMBER" != "" ]; then if [ "$MERGED" == "False" ]; then REF=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['ref']") echo "Setting GitHub Ref to: $REF" - REPO=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['repo']['git_url']") + REPO=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['repo']['clone_url']") echo "Setting GitHub URL to: $REPO" fi fi @@ -70,5 +70,9 @@ LIBGUESTFS_BACKEND=direct virt-customize \ --upload ../virtual-environment.yaml:/home/stack/ \ -a undercloud.qcow2 -popd > /dev/null +# Add performance image scripts +LIBGUESTFS_BACKEND=direct virt-customize --upload ../build_perf_image.sh:/home/stack \ + --upload ../set_perf_images.sh:/home/stack \ + -a undercloud.qcow2 +popd > /dev/null diff --git a/ci/deploy.sh b/ci/deploy.sh index 9cdd8297..966e9864 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -33,6 +33,7 @@ debug="FALSE" declare -i CNT declare UNDERCLOUD declare -A deploy_options_array +declare -a performance_options declare -A NET_MAP SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error) @@ -102,50 +103,29 @@ parse_setting_value() { local mystr=$1 echo $(echo $mystr | grep -Eo "\=.*$" | tr -d '=') } + ##parses network settings yaml into globals parse_network_settings() { - if output=$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py parse_net_settings -n $NETSETS -i $net_isolation_enabled); then - eval "$output" + if local output=$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py parse_net_settings -n $NETSETS -i $net_isolation_enabled); then echo -e "${blue}${output}${reset}" + eval "$output" else + echo -e "${red}ERROR: Failed to parse network settings file $NETSETS ${reset}" exit 1 fi - } -##parses deploy settings yaml into globals and options array -##params: none -##usage: parse_deploy_settings + +##parses deploy settings yaml into globals parse_deploy_settings() { - local global_prefix="deploy_global_params_" - local options_prefix="deploy_deploy_options_" - local myvar myvalue - local settings=$(parse_yaml $DEPLOY_SETTINGS_FILE "deploy_") - - for this_setting in $settings; do - if contains_prefix $this_setting $global_prefix; then - myvar=$(parse_setting_var $this_setting $global_prefix) - if [ -z "$myvar" ]; then - echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}" - fi - myvalue=$(parse_setting_value $this_setting) - # Do not override variables set by cmdline - if [ -z "$(eval echo \$$myvar)" ]; then - eval "$myvar=\$myvalue" - echo -e "${blue}Global parameter set: ${myvar}:${myvalue}${reset}" - else - echo -e "${blue}Global parameter already set: ${myvar}${reset}" - fi - elif contains_prefix $this_setting $options_prefix; then - myvar=$(parse_setting_var $this_setting $options_prefix) - if [ -z "$myvar" ]; then - echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}" - fi - myvalue=$(parse_setting_value $this_setting) - deploy_options_array[$myvar]=$myvalue - echo -e "${blue}Deploy option set: ${myvar}:${myvalue}${reset}" - fi - done + if local output=$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py parse-deploy-settings -f $DEPLOY_SETTINGS_FILE); then + echo -e "${blue}${output}${reset}" + eval "$output" + else + echo -e "${red}ERROR: Failed to parse deploy settings file $DEPLOY_SETTINGS_FILE ${reset}" + exit 1 + fi } + ##parses baremetal yaml settings into compatible json ##writes the json to $CONFIG/instackenv_tmp.json ##params: none @@ -784,12 +764,12 @@ function undercloud_prep_overcloud_deploy { elif [ "${deploy_options_array['sdn_controller']}" == 'opencontrail' ]; then echo -e "${red}ERROR: OpenContrail is currently unsupported...exiting${reset}" exit 1 - elif [[ -z "${deploy_options_array['sdn_controller']}" || "${deploy_options_array['sdn_controller']}" == 'false' ]]; then + elif [[ -z "${deploy_options_array['sdn_controller']}" || "${deploy_options_array['sdn_controller']}" == 'False' ]]; then echo -e "${blue}INFO: SDN Controller disabled...will deploy nosdn scenario${reset}" SDN_IMAGE=opendaylight else echo "${red}Invalid sdn_controller: ${deploy_options_array['sdn_controller']}${reset}" - echo "${red}Valid choices are opendaylight, opendaylight-external, onos, opencontrail, false, or null${reset}" + echo "${red}Valid choices are opendaylight, opendaylight-external, onos, opencontrail, False, or null${reset}" exit 1 fi @@ -805,6 +785,17 @@ function undercloud_prep_overcloud_deploy { ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "rm -f overcloud-full.qcow2" scp ${SSH_OPTIONS[@]} $RESOURCES/overcloud-full-${SDN_IMAGE}.qcow2 "stack@$UNDERCLOUD":overcloud-full.qcow2 + # Push performance options to subscript to modify per-role images as needed + for option in "${performance_options[@]}" ; do + echo -e "${blue}Setting performance option $option${reset}" + ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "bash build_perf_image.sh $option" + done + + # Add performance deploy options if they have been set + if [ ! -z "${deploy_options_array['performance']}" ]; then + DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml" + fi + # make sure ceph is installed DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" @@ -856,6 +847,9 @@ source stackrc set -o errexit echo "Uploading overcloud glance images" openstack overcloud image upload + +bash -x set_perf_images.sh ${performance_roles} + echo "Configuring undercloud and discovering nodes" openstack baremetal import --json instackenv.json openstack baremetal configure boot @@ -1182,6 +1176,7 @@ main() { exit 1 fi if [ -n "$DEPLOY_SETTINGS_FILE" ]; then + echo -e "${blue}INFO: Parsing deploy settings file...${reset}" parse_deploy_settings fi setup_undercloud_vm diff --git a/config/deploy/os-nosdn-performance-ha.yaml b/config/deploy/os-nosdn-performance-ha.yaml new file mode 100644 index 00000000..f7312ad6 --- /dev/null +++ b/config/deploy/os-nosdn-performance-ha.yaml @@ -0,0 +1,23 @@ +global_params: + ha_enabled: true + +deploy_options: + sdn_controller: false + sdn_l3: false + tacker: false + congress: false + sfc: false + vpn: false + performance: + Controller: + kernel: + isolcpus: 1 + hugepage: 2M + intel_iommu: 'on' + Compute: + nova: + libvirtpin: 1 + kernel: + isolcpus: 0 + hugepage: 2M + intel_iommu: 'on' diff --git a/lib/python/apex-python-utils.py b/lib/python/apex-python-utils.py index 1d5b4a7c..7e947ea6 100755 --- a/lib/python/apex-python-utils.py +++ b/lib/python/apex-python-utils.py @@ -7,7 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - import argparse import sys import apex @@ -21,6 +20,10 @@ def parse_net_settings(settings_args): settings_args.network_isolation) settings.dump_bash() +def parse_deploy_settings(settings_args): + settings = apex.DeploySettings(settings_args.path) + settings.dump_bash() + def find_ip(int_args): interface = apex.ip_utils.get_interface(int_args.interface, @@ -73,6 +76,12 @@ nic_template.add_argument('-af', '--address_family', type=int, default=4, help='IP address family') nic_template.set_defaults(func=build_nic_template) +deploy_settings = subparsers.add_parser('parse-deploy-settings', + help='Parse deploy settings file') +deploy_settings.add_argument('-f', '--path', default='deploy_settings.yaml', + help='path to deploy settings file') +deploy_settings.set_defaults(func=parse_deploy_settings) + args = parser.parse_args(sys.argv[1:]) if args.DEBUG: logging.basicConfig(level=logging.DEBUG) diff --git a/lib/python/apex/__init__.py b/lib/python/apex/__init__.py index 88b066b2..2efc64f4 100644 --- a/lib/python/apex/__init__.py +++ b/lib/python/apex/__init__.py @@ -9,3 +9,4 @@ from .net_env import NetworkSettings +from .deploy_env import DeploySettings diff --git a/lib/python/apex/deploy_env.py b/lib/python/apex/deploy_env.py new file mode 100644 index 00000000..5c733248 --- /dev/null +++ b/lib/python/apex/deploy_env.py @@ -0,0 +1,149 @@ +############################################################################## +# Copyright (c) 2016 Michael Chapman (michapma@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +import yaml +import logging + +REQ_DEPLOY_SETTINGS = ['sdn_controller', + 'sdn_l3', + 'tacker', + 'congress', + 'sfc', + 'vpn'] + +OPT_DEPLOY_SETTINGS = ['performance'] + +VALID_ROLES = ['Controller', 'Compute', 'ObjectStorage'] +VALID_PERF_OPTS = ['kernel','nova'] + +class DeploySettings: + """ + This class parses a APEX deploy settings yaml file into an object + + Currently the parsed object is dumped into a bash global definition file + for deploy.sh consumption. This object will later be used directly as + deployment script move to python. + """ + def __init__(self, filename): + with open(filename, 'r') as settings_file: + self.deploy_settings = yaml.load(settings_file) + self._validate_settings() + + def _validate_settings(self): + """ + Validates the deploy settings file provided + + DeploySettingsException will be raised if validation fails. + """ + + if 'deploy_options' not in self.deploy_settings: + raise DeploySettingsException("No deploy options provided in" + "deploy settings file") + if 'global_params' not in self.deploy_settings: + raise DeploySettingsException("No global options provided in" + "deploy settings file") + + deploy_options = self.deploy_settings['deploy_options'] + if not isinstance(deploy_options, dict): + raise DeploySettingsException("deploy_options should be a list") + + for option in deploy_options: + if option not in REQ_DEPLOY_SETTINGS + OPT_DEPLOY_SETTINGS: + raise DeploySettingsException("Invalid deploy_option {} " + "specified".format(option)) + + for required_setting in REQ_DEPLOY_SETTINGS: + if required_setting not in deploy_options: + self.deploy_settings['deploy_options'][required] = False + + if 'performance' in deploy_options: + if not isinstance(deploy_options['performance'], dict): + raise DeploySettingsException("Performance deploy_option" + "must be a dictionary.") + for role,role_perf_sets in deploy_options['performance'].items(): + if role not in VALID_ROLES: + raise DeploySettingsException("Performance role {}" + "is not valid, choose" + "from {}".format( + role," ".join(VALID_ROLES) + )) + + for key in role_perf_sets: + if key not in VALID_PERF_OPTS: + raise DeploySettingsException("Performance option {}" + "is not valid, choose" + "from {}".format( + key," ".join( + VALID_PERF_OPTS))) + + + def _dump_performance(self): + """ + Creates performance settings string for bash consumption. + + Output will be in the form of a list that can be iterated over in bash, + with each string being the direct input to the performance setting script + in the form to facilitate modification of the + correct image. + """ + bash_str = 'performance_options=(\n' + for role,settings in self.deploy_settings['deploy_options']['performance'].items(): + for category,options in settings.items(): + for key,value in options.items(): + bash_str += "\"{} {} {} {}\"\n".format(role, category, key, value) + bash_str += ')\n' + bash_str += '\n' + bash_str += 'performance_roles=(\n' + for role in self.deploy_settings['deploy_options']['performance']: + bash_str += role + '\n' + bash_str += ')\n' + bash_str += '\n' + + return bash_str + + def _dump_deploy_options_array(self): + """ + Creates deploy settings array in bash syntax. + """ + bash_str = '' + for key,value in self.deploy_settings['deploy_options'].items(): + if not isinstance(value, bool): + bash_str += "deploy_options_array[{}]=\"{}\"\n".format(key, value) + else: + bash_str += "deploy_options_array[{}]={}\n".format(key, value) + return bash_str + + def dump_bash(self, path=None): + """ + Prints settings for bash consumption. + + If optional path is provided, bash string will be written to the file + instead of stdout. + """ + bash_str = '' + for key, value in self.deploy_settings['global_params'].items(): + bash_str += "if [ -z \"$(eval echo \$${})\" ]; then\n{}={}\nfi\n".format(key,key, value) + if 'performance' in self.deploy_settings['deploy_options']: + bash_str += self._dump_performance() + bash_str += self._dump_deploy_options_array() + + if path: + with open(path, 'w') as file: + file.write(bash_str) + else: + print(bash_str) + + +class DeploySettingsException(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value -- cgit 1.2.3-korg