Fixes OVS DPDK scenarios

This fixes nosdn OVS DPDK scenarios to work with OVS 2.6.  It also adds
support for the ODL DPDK scenario.  OVS DPDK configuration is done
during preconfig on compute nodes.  All nodes make use of first boot
script to get kernel params like hugepages.  The network settings are
modified to use real nic names for the virtual network settings file.
This is required due to a timing bug with os-net-config and using
first-boot script.  The network settings file also contains the UIO
driver to use with DPDK.

JIRA: APEX-314

opnfv-tht-pr: 117

Change-Id: I46b6fca71ecec38981968133ba2411f64d7c6445
Signed-off-by: Tim Rozet <trozet@redhat.com>

7 files changed, 134 insertions, 15 deletions

diff --git a/build/network-environment.yaml b/build/network-environment.yaml
index 8367371b..40f8e3be 100644
--- a/build/network-environment.yaml
+++ b/build/network-environment.yaml
@@ -44,6 +44,10 @@ resource_registry:
   OS::TripleO::Services::SwiftRingBuilder: OS::Heat::None
   OS::TripleO::Services::SwiftProxy: OS::Heat::None
 
+  # Extra Config
+  OS::TripleO::ComputeExtraConfigPre: OS::Heat::None
+  OS::TripleO::ControllerExtraConfigPre: OS::Heat::None
+
 parameter_defaults:
   NeutronExternalNetworkBridge: 'br-ex'
 
diff --git a/build/nics-template.yaml.jinja2 b/build/nics-template.yaml.jinja2
index f2c8000a..0913fa98 100644
--- a/build/nics-template.yaml.jinja2
+++ b/build/nics-template.yaml.jinja2
@@ -156,7 +156,7 @@ resources:
             {%- if nets['tenant']['enabled'] and nets['tenant']['nic_mapping'][role]['vlan'] == 'native' %}
             {%- if ovs_dpdk_bridge == 'br-phy' and role == 'compute' %}
             -
-              type: ovs_bridge
+              type: ovs_user_bridge
               name: {{ ovs_dpdk_bridge }}
               use_dhcp: false
               addresses:
@@ -164,14 +164,15 @@ resources:
                   ip_netmask: {get_param: TenantIpSubnet}
               members:
                 -
-                  type: interface
-                  name: {{ nets['tenant']['nic_mapping'][role]['members'][0] }}
-                  # force the MAC address of the bridge to this interface
-                  primary: true
-            -
-              type: ovs_bridge
-              name: br-tun
-              use_dhcp: false
+                  type: ovs_dpdk_port
+                  name: dpdk0
+                  driver: {{ nets['tenant']['nic_mapping'][role]['uio_driver'] }}
+                  members:
+                    -
+                      type: interface
+                      name: {{ nets['tenant']['nic_mapping'][role]['members'][0] }}
+                      # force the MAC address of the bridge to this interface
+                      primary: true
             {%- else %}
             -
               type: {{ nets['tenant']['nic_mapping'][role]['phys_type'] }}
@@ -208,7 +209,11 @@ resources:
                   next_hop: {get_param: ExternalInterfaceDefaultRoute}
             {%- elif nets['external'][0]['enabled'] and external_net_type == 'br-ex' and nets['external'][0]['nic_mapping'][role]['vlan'] == 'native' %}
             -
+              {%- if ovs_dpdk_bridge == 'br-phy' and role == 'compute' %}
+              type: ovs_user_bridge
+              {%- else %}
               type: ovs_bridge
+              {%- endif %}
               name: {get_input: bridge_name}
               use_dhcp: false
               members:
diff --git a/build/opnfv-environment.yaml b/build/opnfv-environment.yaml
index 8b58b359..b42779ba 100644
--- a/build/opnfv-environment.yaml
+++ b/build/opnfv-environment.yaml
@@ -13,6 +13,9 @@ parameter_defaults:
   #NovaSchedulerDefaultFilters: "RamFilter,ComputeFilter,AvailabilityZoneFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,NUMATopologyFilter"
   # Kernel arguments, this value will be set to kernel arguments specified for compute nodes in deploy setting file.
   #ComputeKernelArgs: "intel_iommu=on iommu=pt default_hugepagesz=2MB hugepagesz=2MB hugepages=2048"
+  #PmdCoreList: 1
+  #OvsDpdkCoreList: 2
+  #OvsDpdkSocketMemory: 1024
   ExtraConfig:
     tripleo::ringbuilder::build_ring: False
     nova::nova_public_key:
diff --git a/build/ovs-dpdk-preconfig.yaml b/build/ovs-dpdk-preconfig.yaml
new file mode 100644
index 00000000..d57b0b7a
--- /dev/null
+++ b/build/ovs-dpdk-preconfig.yaml
@@ -0,0 +1,101 @@
+heat_template_version: 2014-10-16
+
+description: >
+  Example extra config for post-deployment
+
+parameters:
+  server:
+    type: string
+  OvsDpdkCoreList:
+    description: >
+      List of logical cores for OVS DPDK
+    type: string
+    default: ""
+  OvsDpdkSocketMemory:
+    description: Memory allocated for each socket
+    default: ""
+    type: string
+  PmdCoreList:
+    description: >
+      A list or range of physical CPU cores to be pinned to PMD
+      The given args will be appended to the tuned cpu-partitioning profile.
+      Ex. HostCpusList: '4-12' will tune cores from 4-12
+    type: string
+    default: ""
+
+resources:
+  OvsDpdkSetup:
+    type: OS::Heat::StructuredDeployment
+    properties:
+      server:  {get_param: server}
+      config: {get_resource: OvsDpdkConfig}
+
+  OvsDpdkConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        str_replace:
+          template: |
+            #!/bin/bash
+            set -x
+            get_mask()
+            {
+              local list=$1
+              local mask=0
+              declare -a bm
+              max_idx=0
+              for core in $(echo $list | sed 's/,/ /g')
+              do
+                  index=$(($core/32))
+                  bm[$index]=0
+                  if [ $max_idx -lt $index ]; then
+                     max_idx=$index
+                  fi
+              done
+              for ((i=$max_idx;i>=0;i--));
+              do
+                  bm[$i]=0
+              done
+              for core in $(echo $list | sed 's/,/ /g')
+              do
+                  index=$(($core/32))
+                  temp=$((1<<$core))
+                  bm[$index]=$((${bm[$index]} | $temp))
+              done
+              printf -v mask "%x" "${bm[$max_idx]}"
+              for ((i=$max_idx-1;i>=0;i--));
+              do
+                  printf -v hex "%08x" "${bm[$i]}"
+                  mask+=$hex
+              done
+              printf "%s" "$mask"
+            }
+            pmd_cpu_mask=$( get_mask $PMD_CORES )
+            dpdk_lcore_mask=$( get_mask $DPDK_CORES )
+            yum remove -y vpp-devel
+            yum install -y /root/dpdk_rpms/*
+            systemctl restart openvswitch
+            sleep 5
+            sed  -i "s/#user\s*=.*/user = \"root\"/" /etc/libvirt/qemu.conf
+            sed  -i "s/#group\s*=.*/group = \"root\"/" /etc/libvirt/qemu.conf
+            ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
+            if [ -n "$SOCKET_MEMORY" ]; then
+              ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$SOCKET_MEMORY
+            fi
+            if [ -n "$pmd_cpu_mask" ]; then
+              ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=$pmd_cpu_mask
+            fi
+            if [ -n "$dpdk_lcore_mask" ]; then
+              ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=$dpdk_lcore_mask
+            fi
+            systemctl restart openvswitch
+
+          params:
+            $DPDK_CORES: {get_param: OvsDpdkCoreList}
+            $PMD_CORES: {get_param: PmdCoreList}
+            $SOCKET_MEMORY: {get_param: OvsDpdkSocketMemory}
+outputs:
+  deploy_stdout:
+    description: Output of the extra dpdk ovs  deployment
+    value: {get_attr: [OvsDpdkSetup, deploy_stdout]}
diff --git a/build/rpm_specs/opnfv-apex-common.spec b/build/rpm_specs/opnfv-apex-common.spec
index 6a03be04..4266897c 100644
--- a/build/rpm_specs/opnfv-apex-common.spec
+++ b/build/rpm_specs/opnfv-apex-common.spec
@@ -56,6 +56,8 @@ install config/deploy/os-odl_l3-fdio-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-
 install config/deploy/os-odl_l3-fdio-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl_l3-fdio-ha.yaml
 install config/deploy/os-odl_l3-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl_l3-nofeature-ha.yaml
 install config/deploy/os-odl_l3-nofeature-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl_l3-nofeature-noha.yaml
+install config/deploy/os-odl_l3-ovs-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl_l3-ovs-ha.yaml
+install config/deploy/os-odl_l3-ovs-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl_l3-ovs-noha.yaml
 install config/deploy/os-odl-gluon-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-gluon-noha.yaml
 #install config/deploy/os-onos-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-onos-nofeature-ha.yaml
 #install config/deploy/os-onos-sfc-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-onos-sfc-ha.yaml
@@ -137,6 +139,8 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %{_sysconfdir}/opnfv-apex/os-odl_l2-fdio-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-odl_l3-fdio-noha.yaml
 %{_sysconfdir}/opnfv-apex/os-odl_l3-fdio-ha.yaml
+%{_sysconfdir}/opnfv-apex/os-odl_l3-ovs-noha.yaml
+%{_sysconfdir}/opnfv-apex/os-odl_l3-ovs-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-odl_l3-nofeature-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-odl_l3-nofeature-noha.yaml
 %{_sysconfdir}/opnfv-apex/os-odl-gluon-noha.yaml
@@ -157,6 +161,8 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %doc %{_docdir}/opnfv/inventory.yaml.example
 
 %changelog
+* Thu Mar 23 2017 Tim Rozet <trozet@redhat.com> - 4.0-8
+- Adds os-odl_l3-ovs-ha and noha scenarios
 * Mon Mar 12 2017 Feng Pan <fpan@redhat.com> - 4.0-7
 - Add os-nosdn-fdio-ha.yaml
 * Fri Mar 10 2017 Feng Pan <fpan@redhat.com> - 4.0-6
diff --git a/build/undercloud.sh b/build/undercloud.sh
index 424f6f86..e3194e2b 100755
--- a/build/undercloud.sh
+++ b/build/undercloud.sh
@@ -53,6 +53,7 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     --upload ${BUILD_ROOT}/first-boot.yaml:/home/stack/ \
     --upload ${BUILD_ROOT}/kvm4nfv-1st-boot.yaml:/home/stack/ \
     --upload ${BUILD_DIR}/enable_rt_kvm.yaml:/home/stack/ \
+    --upload ${BUILD_ROOT}/ovs-dpdk-preconfig.yaml:/home/stack/ \
     --upload ${BUILD_ROOT}/csit-environment.yaml:/home/stack/ \
     --upload ${BUILD_ROOT}/virtual-environment.yaml:/home/stack/ \
     --install "python2-congressclient" \
diff --git a/build/variables.sh b/build/variables.sh
index 9322442e..5249c2ec 100644
--- a/build/variables.sh
+++ b/build/variables.sh
@@ -24,13 +24,12 @@ onos_ovs_pkg=package_ovs_rpm3.tar.gz
 if [ -z ${GS_PATHNAME+x} ]; then
     GS_PATHNAME=/colorado
 fi
-dpdk_uri_base=http://artifacts.opnfv.org/ovsnfv$GS_PATHNAME
+dpdk_uri_base=http://artifacts.opnfv.org/ovsnfv
 dpdk_rpms=(
-'ovs4opnfv-e8acab14-dpdk-16.04.0-1.el7.centos.x86_64.rpm'
-'ovs4opnfv-e8acab14-dpdk-devel-16.04.0-1.el7.centos.x86_64.rpm'
-'ovs4opnfv-e8acab14-dpdk-examples-16.04.0-1.el7.centos.x86_64.rpm'
-'ovs4opnfv-e8acab14-dpdk-tools-16.04.0-1.el7.centos.x86_64.rpm'
-'ovs4opnfv-e8acab14-openvswitch-2.5.90-0.12032.gitc61e93d6.1.el7.centos.x86_64.rpm'
+'ovs4opnfv-e8acab14-dpdk-16.11-5.el7.centos.x86_64.rpm'
+'ovs4opnfv-e8acab14-dpdk-devel-16.11-5.el7.centos.x86_64.rpm'
+'ovs4opnfv-e8acab14-dpdk-examples-16.11-5.el7.centos.x86_64.rpm'
+'ovs4opnfv-e8acab14-dpdk-tools-16.11-5.el7.centos.x86_64.rpm'
 )
 
 kvmfornfv_uri_base="http://artifacts.opnfv.org/kvmfornfv"