Add pinning support

Add support for CPU pinning of libvirt guests and also
restricting host processes to selected CPUs via isolcpus.

Hugepage support is added using the same mechanism as
isolcpus, along with a perf scenario where all 3 performance
options are enabled.

Deploy options are now parsed in python

JIRA: APEX-127
JIRA: APEX-105
JIRA: APEX-106

Change-Id: I438e80fb88e596cc017595d43bc1efda1001325c
opnfv-tht-pr: 8
Signed-off-by: Michael Chapman <woppin@gmail.com>

10 files changed, 325 insertions, 40 deletions

diff --git a/build/build_perf_image.sh b/build/build_perf_image.sh
new file mode 100644
index 00000000..0a3e3d0f
--- /dev/null
+++ b/build/build_perf_image.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 Red Hat Inc.
+# Michael Chapman <michapma@redhat.com>
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+ROLE=$1
+shift
+CATEGORY=$1
+shift
+KEY=$1
+shift
+VALUE=$1
+shift
+
+IMAGE=$ROLE-overcloud-full.qcow2
+
+# Create image copy for this role
+if [ ! -f $IMAGE ] ; then
+  cp overcloud-full.qcow2 $IMAGE
+fi
+
+if [ "$CATEGORY" == "nova" ]; then
+  if [ "$KEY" == "libvirtpin" ]; then
+    sudo sed -i "s/#LibvirtCPUPinSet:.*/LibvirtCPUPinSet: '${VALUE}'/" /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml
+  fi
+fi
+
+if [ "$CATEGORY" == "kernel" ]; then
+  LIBGUESTFS_BACKEND=direct virt-customize \
+    --run-command "bash -x /root/setkernelparam.sh $KEY $VALUE" \
+    -a $IMAGE
+fi
+
diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh
index 065201f8..d0ee3f01 100755
--- a/build/overcloud-full.sh
+++ b/build/overcloud-full.sh
@@ -29,5 +29,13 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     --run-command "echo 'nf_conntrack_proto_sctp' > /etc/modules-load.d/nf_conntrack_proto_sctp.conf" \
     -a overcloud-full_build.qcow2
 
+###################################
+#####  Add CPU pinning script #####
+###################################
+
+LIBGUESTFS_BACKEND=direct virt-customize \
+    --upload ../setkernelparam.sh:/root \
+    -a overcloud-full_build.qcow2
+
 mv -f overcloud-full_build.qcow2 overcloud-full.qcow2
 popd > /dev/null
diff --git a/build/set_perf_images.sh b/build/set_perf_images.sh
new file mode 100644
index 00000000..ea31c7fa
--- /dev/null
+++ b/build/set_perf_images.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2016 Red Hat Inc.
+# Michael Chapman <michapma@redhat.com>
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+for ROLE in $@; do
+  if [ -f $ROLE-overcloud-full.qcow2 ]; then
+    echo "Uploading $ROLE-overcloud-full.qcow2 "
+    KERNEL=$(glance image-show overcloud-full | grep 'kernel_id' | cut -d '|' -f 3 | xargs)
+    RAMDISK=$(glance image-show overcloud-full | grep 'ramdisk_id' | cut -d '|' -f 3 | xargs)
+    glance image-create --name $ROLE-overcloud-full --disk-format qcow2 --file $ROLE-overcloud-full.qcow2 --container-format bare --property ramdisk_id=$RAMDISK --property kernel_id=$KERNEL
+  fi
+
+  if [ "$ROLE" == "Controller" ]; then
+    sed -i "s/overcloud-full/Controller-overcloud-full" opnfv-environment.yaml
+  fi
+
+  if [ "$ROLE" == "Compute" ]; then
+    sudo sed -i "s/NovaImage: overcloud-full/Compute-overcloud-full/" /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml
+  fi
+
+  if [ "$ROLE" == "BlockStorage" ]; then
+    sudo sed -i "s/BlockStorageImage: overcloud-full/BlockStorage-overcloud-full/" /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml
+  fi
+done
diff --git a/build/setkernelparam.sh b/build/setkernelparam.sh
new file mode 100644
index 00000000..b6986d6c
--- /dev/null
+++ b/build/setkernelparam.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2016 Red Hat Inc.
+# Michael Chapman <michapma@redhat.com>
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+GRUBCONF='/boot/grub2/grub.conf'
+
+if [ "$1" == "" ]; then
+  echo "No kernel parameter name provided, not modifying grub.conf"
+  exit 1
+fi
+
+if [ "$2" == "" ]; then
+  echo "No kernel parameter value provided, not modifying grub.conf"
+  exit 1
+fi
+
+echo "Setting $1=$2 in $GRUBCONF"
+echo "GRUB_CMDLINE_LINUX=\"\$GRUB_CMDLINE_LINUX $1=$2\"" >> /etc/default/grub
+grub2-mkconfig > $GRUBCONF
+exit 0
diff --git a/build/undercloud.sh b/build/undercloud.sh
index ed4f2b34..04c2667f 100755
--- a/build/undercloud.sh
+++ b/build/undercloud.sh
@@ -43,7 +43,7 @@ if [ "$PR_NUMBER" != "" ]; then
   if [ "$MERGED" == "False" ]; then
     REF=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['ref']")
     echo "Setting GitHub Ref to: $REF"
-    REPO=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['repo']['git_url']")
+    REPO=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['repo']['clone_url']")
     echo "Setting GitHub URL to: $REPO"
   fi
 fi
@@ -70,5 +70,9 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     --upload ../virtual-environment.yaml:/home/stack/ \
     -a undercloud.qcow2
 
-popd > /dev/null
+# Add performance image scripts
+LIBGUESTFS_BACKEND=direct virt-customize --upload ../build_perf_image.sh:/home/stack \
+                                         --upload ../set_perf_images.sh:/home/stack \
+                                         -a undercloud.qcow2
 
+popd > /dev/null
diff --git a/ci/deploy.sh b/ci/deploy.sh
index 9cdd8297..966e9864 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -33,6 +33,7 @@ debug="FALSE"
 declare -i CNT
 declare UNDERCLOUD
 declare -A deploy_options_array
+declare -a performance_options
 declare -A NET_MAP
 
 SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error)
@@ -102,50 +103,29 @@ parse_setting_value() {
   local mystr=$1
   echo $(echo $mystr | grep -Eo "\=.*$" | tr -d '=')
 }
+
 ##parses network settings yaml into globals
 parse_network_settings() {
-  if output=$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py parse_net_settings -n $NETSETS -i $net_isolation_enabled); then
-      eval "$output"
+  if local output=$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py parse_net_settings -n $NETSETS -i $net_isolation_enabled); then
       echo -e "${blue}${output}${reset}"
+      eval "$output"
   else
+      echo -e "${red}ERROR: Failed to parse network settings file $NETSETS ${reset}"
       exit 1
   fi
-
 }
-##parses deploy settings yaml into globals and options array
-##params: none
-##usage:  parse_deploy_settings
+
+##parses deploy settings yaml into globals
 parse_deploy_settings() {
-  local global_prefix="deploy_global_params_"
-  local options_prefix="deploy_deploy_options_"
-  local myvar myvalue
-  local settings=$(parse_yaml $DEPLOY_SETTINGS_FILE "deploy_")
-
-  for this_setting in $settings; do
-    if contains_prefix $this_setting $global_prefix; then
-      myvar=$(parse_setting_var $this_setting $global_prefix)
-      if [ -z "$myvar" ]; then
-        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
-      fi
-      myvalue=$(parse_setting_value $this_setting)
-      # Do not override variables set by cmdline
-      if [ -z "$(eval echo \$$myvar)" ]; then
-        eval "$myvar=\$myvalue"
-        echo -e "${blue}Global parameter set: ${myvar}:${myvalue}${reset}"
-      else
-        echo -e "${blue}Global parameter already set: ${myvar}${reset}"
-      fi
-    elif contains_prefix $this_setting $options_prefix; then
-      myvar=$(parse_setting_var $this_setting $options_prefix)
-      if [ -z "$myvar" ]; then
-        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
-      fi
-      myvalue=$(parse_setting_value $this_setting)
-      deploy_options_array[$myvar]=$myvalue
-      echo -e "${blue}Deploy option set: ${myvar}:${myvalue}${reset}"
-    fi
-  done
+  if local output=$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py parse-deploy-settings -f $DEPLOY_SETTINGS_FILE); then
+      echo -e "${blue}${output}${reset}"
+      eval "$output"
+  else
+      echo -e "${red}ERROR: Failed to parse deploy settings file $DEPLOY_SETTINGS_FILE ${reset}"
+      exit 1
+  fi
 }
+
 ##parses baremetal yaml settings into compatible json
 ##writes the json to $CONFIG/instackenv_tmp.json
 ##params: none
@@ -784,12 +764,12 @@ function undercloud_prep_overcloud_deploy {
   elif [ "${deploy_options_array['sdn_controller']}" == 'opencontrail' ]; then
     echo -e "${red}ERROR: OpenContrail is currently unsupported...exiting${reset}"
     exit 1
-  elif [[ -z "${deploy_options_array['sdn_controller']}" || "${deploy_options_array['sdn_controller']}" == 'false' ]]; then
+  elif [[ -z "${deploy_options_array['sdn_controller']}" || "${deploy_options_array['sdn_controller']}" == 'False' ]]; then
     echo -e "${blue}INFO: SDN Controller disabled...will deploy nosdn scenario${reset}"
     SDN_IMAGE=opendaylight
   else
     echo "${red}Invalid sdn_controller: ${deploy_options_array['sdn_controller']}${reset}"
-    echo "${red}Valid choices are opendaylight, opendaylight-external, onos, opencontrail, false, or null${reset}"
+    echo "${red}Valid choices are opendaylight, opendaylight-external, onos, opencontrail, False, or null${reset}"
     exit 1
   fi
 
@@ -805,6 +785,17 @@ function undercloud_prep_overcloud_deploy {
   ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "rm -f overcloud-full.qcow2"
   scp ${SSH_OPTIONS[@]} $RESOURCES/overcloud-full-${SDN_IMAGE}.qcow2 "stack@$UNDERCLOUD":overcloud-full.qcow2
 
+  # Push performance options to subscript to modify per-role images as needed
+  for option in "${performance_options[@]}" ; do
+    echo -e "${blue}Setting performance option $option${reset}"
+    ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "bash build_perf_image.sh $option"
+  done
+
+  # Add performance deploy options if they have been set
+  if [ ! -z "${deploy_options_array['performance']}" ]; then
+    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/numa.yaml"
+  fi
+
   # make sure ceph is installed
   DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml"
 
@@ -856,6 +847,9 @@ source stackrc
 set -o errexit
 echo "Uploading overcloud glance images"
 openstack overcloud image upload
+
+bash -x set_perf_images.sh ${performance_roles}
+
 echo "Configuring undercloud and discovering nodes"
 openstack baremetal import --json instackenv.json
 openstack baremetal configure boot
@@ -1182,6 +1176,7 @@ main() {
     exit 1
   fi
   if [ -n "$DEPLOY_SETTINGS_FILE" ]; then
+    echo -e "${blue}INFO: Parsing deploy settings file...${reset}"
     parse_deploy_settings
   fi
   setup_undercloud_vm
diff --git a/config/deploy/os-nosdn-performance-ha.yaml b/config/deploy/os-nosdn-performance-ha.yaml
new file mode 100644
index 00000000..f7312ad6
--- /dev/null
+++ b/config/deploy/os-nosdn-performance-ha.yaml
@@ -0,0 +1,23 @@
+global_params:
+  ha_enabled: true
+
+deploy_options:
+  sdn_controller: false
+  sdn_l3: false
+  tacker: false
+  congress: false
+  sfc: false
+  vpn: false
+  performance:
+    Controller:
+      kernel:
+        isolcpus: 1
+        hugepage: 2M
+        intel_iommu: 'on'
+    Compute:
+      nova:
+        libvirtpin: 1
+      kernel:
+        isolcpus: 0
+        hugepage: 2M
+        intel_iommu: 'on'
diff --git a/lib/python/apex-python-utils.py b/lib/python/apex-python-utils.py
index 1d5b4a7c..7e947ea6 100755
--- a/lib/python/apex-python-utils.py
+++ b/lib/python/apex-python-utils.py
@@ -7,7 +7,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-
 import argparse
 import sys
 import apex
@@ -21,6 +20,10 @@ def parse_net_settings(settings_args):
                                     settings_args.network_isolation)
     settings.dump_bash()
 
+def parse_deploy_settings(settings_args):
+    settings = apex.DeploySettings(settings_args.path)
+    settings.dump_bash()
+
 
 def find_ip(int_args):
     interface = apex.ip_utils.get_interface(int_args.interface,
@@ -73,6 +76,12 @@ nic_template.add_argument('-af', '--address_family', type=int, default=4,
                           help='IP address family')
 nic_template.set_defaults(func=build_nic_template)
 
+deploy_settings = subparsers.add_parser('parse-deploy-settings',
+                                        help='Parse deploy settings file')
+deploy_settings.add_argument('-f', '--path', default='deploy_settings.yaml',
+                             help='path to deploy settings file')
+deploy_settings.set_defaults(func=parse_deploy_settings)
+
 args = parser.parse_args(sys.argv[1:])
 if args.DEBUG:
     logging.basicConfig(level=logging.DEBUG)
diff --git a/lib/python/apex/__init__.py b/lib/python/apex/__init__.py
index 88b066b2..2efc64f4 100644
--- a/lib/python/apex/__init__.py
+++ b/lib/python/apex/__init__.py
@@ -9,3 +9,4 @@
 
 
 from .net_env import NetworkSettings
+from .deploy_env import DeploySettings
diff --git a/lib/python/apex/deploy_env.py b/lib/python/apex/deploy_env.py
new file mode 100644
index 00000000..5c733248
--- /dev/null
+++ b/lib/python/apex/deploy_env.py
@@ -0,0 +1,149 @@
+##############################################################################
+# Copyright (c) 2016 Michael Chapman (michapma@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import yaml
+import logging
+
+REQ_DEPLOY_SETTINGS = ['sdn_controller',
+                       'sdn_l3',
+                       'tacker',
+                       'congress',
+                       'sfc',
+                       'vpn']
+
+OPT_DEPLOY_SETTINGS = ['performance']
+
+VALID_ROLES = ['Controller', 'Compute', 'ObjectStorage']
+VALID_PERF_OPTS = ['kernel','nova']
+
+class DeploySettings:
+    """
+    This class parses a APEX deploy settings yaml file into an object
+
+    Currently the parsed object is dumped into a bash global definition file
+    for deploy.sh consumption. This object will later be used directly as
+    deployment script move to python.
+    """
+    def __init__(self, filename):
+        with open(filename, 'r') as settings_file:
+            self.deploy_settings = yaml.load(settings_file)
+            self._validate_settings()
+
+    def _validate_settings(self):
+        """
+        Validates the deploy settings file provided
+
+        DeploySettingsException will be raised if validation fails.
+        """
+
+        if 'deploy_options' not in self.deploy_settings:
+            raise DeploySettingsException("No deploy options provided in"
+                                          "deploy settings file")
+        if 'global_params' not in self.deploy_settings:
+            raise DeploySettingsException("No global options provided in"
+                                          "deploy settings file")
+
+        deploy_options = self.deploy_settings['deploy_options']
+        if not isinstance(deploy_options, dict):
+            raise DeploySettingsException("deploy_options should be a list")
+
+        for option in deploy_options:
+            if option not in REQ_DEPLOY_SETTINGS + OPT_DEPLOY_SETTINGS:
+                raise DeploySettingsException("Invalid deploy_option {} "
+                                              "specified".format(option))
+
+        for required_setting in REQ_DEPLOY_SETTINGS:
+            if required_setting not in deploy_options:
+                self.deploy_settings['deploy_options'][required] = False
+
+        if 'performance' in deploy_options:
+            if not isinstance(deploy_options['performance'], dict):
+                raise DeploySettingsException("Performance deploy_option"
+                                              "must be a dictionary.")
+            for role,role_perf_sets in deploy_options['performance'].items():
+                if role not in VALID_ROLES:
+                    raise DeploySettingsException("Performance role {}"
+                                                  "is not valid, choose"
+                                                  "from {}".format(
+                                                  role," ".join(VALID_ROLES)
+                                                  ))
+
+                for key in role_perf_sets:
+                    if key not in VALID_PERF_OPTS:
+                        raise DeploySettingsException("Performance option {}"
+                                                      "is not valid, choose"
+                                                      "from {}".format(
+                                                      key," ".join(
+                                                      VALID_PERF_OPTS)))
+
+
+    def _dump_performance(self):
+        """
+        Creates performance settings string for bash consumption.
+
+        Output will be in the form of a list that can be iterated over in bash,
+        with each string being the direct input to the performance setting script
+        in the form <role> <category> <key> <value> to facilitate modification of the
+        correct image.
+        """
+        bash_str = 'performance_options=(\n'
+        for role,settings in self.deploy_settings['deploy_options']['performance'].items():
+            for category,options in settings.items():
+                for key,value in options.items():
+                    bash_str += "\"{} {} {} {}\"\n".format(role, category, key, value)
+        bash_str += ')\n'
+        bash_str += '\n'
+        bash_str += 'performance_roles=(\n'
+        for role in self.deploy_settings['deploy_options']['performance']:
+            bash_str += role + '\n'
+        bash_str += ')\n'
+        bash_str += '\n'
+
+        return bash_str
+
+    def _dump_deploy_options_array(self):
+        """
+        Creates deploy settings array in bash syntax.
+        """
+        bash_str = ''
+        for key,value in self.deploy_settings['deploy_options'].items():
+            if not isinstance(value, bool):
+                bash_str += "deploy_options_array[{}]=\"{}\"\n".format(key, value)
+            else:
+                bash_str += "deploy_options_array[{}]={}\n".format(key, value)
+        return bash_str
+
+    def dump_bash(self, path=None):
+        """
+        Prints settings for bash consumption.
+
+        If optional path is provided, bash string will be written to the file
+        instead of stdout.
+        """
+        bash_str = ''
+        for key, value in self.deploy_settings['global_params'].items():
+            bash_str += "if [ -z \"$(eval echo \$${})\" ]; then\n{}={}\nfi\n".format(key,key, value)
+        if 'performance' in self.deploy_settings['deploy_options']:
+            bash_str += self._dump_performance()
+        bash_str += self._dump_deploy_options_array()
+
+        if path:
+            with open(path, 'w') as file:
+                file.write(bash_str)
+        else:
+            print(bash_str)
+
+
+class DeploySettingsException(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return self.value