Contrail network realignement + DPDK enablement

This patch moves Contrail roles communication from public/external
to internal_api network for OpenStack API.
It also adds the option to enable dpdk.
Monolithic firstboot script is broken down into small pre-network
and per-node extraconfig scripts

Change-Id: I296a3bf60cef6fa950fd71d6e68effe367d1e66b
Closes-Bug: 1698422

3 files changed, 373 insertions, 0 deletions

diff --git a/extraconfig/all_nodes/contrail/enable_contrail_repo.yaml b/extraconfig/all_nodes/contrail/enable_contrail_repo.yaml
new file mode 100644
index 00000000..51da6f65
--- /dev/null
+++ b/extraconfig/all_nodes/contrail/enable_contrail_repo.yaml
@@ -0,0 +1,43 @@
+heat_template_version: pike
+
+parameters:
+  ContrailRepo:
+    type: string
+    default: ''
+
+resources:
+  userdata:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: {get_resource: EnableContrailRepoConfig}
+
+  EnableContrailRepoConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      config:
+        str_replace:
+          template: |
+            #!/bin/bash
+            contrail_repo=$contrail_repo
+            if [[ ${contrail_repo} ]]; then
+               cat <<EOF > /etc/yum.repos.d/contrail.repo
+            [Contrail]
+            name=Contrail Repo
+            baseurl=${contrail_repo}
+            enabled=1
+            gpgcheck=0
+            protect=1
+            metadata_expire=30
+            EOF
+            fi
+          params:
+            $contrail_repo: {get_param: ContrailRepo}
+
+outputs:
+  # This means get_resource from the parent template will get the userdata, see:
+  # http://docs.openstack.org/developer/heat/template_guide/composition.html#making-your-template-resource-more-transparent
+  # Note this is new-for-kilo, an alternative is returning a value then using
+  # get_attr in the parent template instead.
+  OS::stack_id:
+    value: {get_resource: userdata}
diff --git a/extraconfig/pre_network/contrail/compute_pre_network.yaml b/extraconfig/pre_network/contrail/compute_pre_network.yaml
new file mode 100644
index 00000000..a30330f9
--- /dev/null
+++ b/extraconfig/pre_network/contrail/compute_pre_network.yaml
@@ -0,0 +1,162 @@
+heat_template_version: pike
+
+# NOTE: You don't need to pass the parameter explicitly from the
+# parent template, it can be specified via the parameter_defaults
+# in the resource_registry instead, if you want to override the default
+# and/or share values with other templates in the tree.
+parameters:
+  ContrailRepo:
+    type: string
+    default: ''
+  ContrailVrouterPhysicalInterface:
+    default: 'eth0'
+    description: vRouter physical interface
+    type: string
+  ContrailVlanParentInterface:
+    default: ''
+    description: Parent interface of vlan interface
+    type: string
+  ContrailBondInterface:
+    default: ''
+    description: Bond interface name
+    type: string
+  ContrailBondInterfaceMembers:
+    default: ''
+    description: Bond interface members
+    type: string
+  ContrailBondMode:
+    default: '4'
+    description: Bond Mode
+    type: string
+  ContrailBondPolicy:
+    default: '1'
+    description: Bond Policy
+    type: string
+  RoleParameters:
+    type: json
+    description: Role Specific parameters
+    default: {}
+  ServiceNames:
+    type: comma_delimited_list
+    default: []
+  server:
+    type: string
+
+description: >
+  This template installs the Contrail kernel module  packages in order
+  to bring vhost0 interface up. Vhost0 interface must be up before
+  os-net-config takes over.
+
+resources:
+
+  ContrailVrouterModuleDeployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      name: ContrailVrouterModuleDeployment
+      server:  {get_param: server}
+      config: {get_resource: ContrailVrouterModuleConfig}
+      input_values:
+        phy_int: {get_param: ContrailVrouterPhysicalInterface}
+        bond_int: {get_param: ContrailBondInterface}
+        bond_int_members: {get_param: ContrailBondInterfaceMembers}
+        vlan_parent: {get_param: ContrailVlanParentInterface}
+        contrail_repo: {get_param: ContrailRepo}
+        bond_mode: {get_param: ContrailBondMode}
+        bond_policy: {get_param: ContrailBondPolicy}
+      actions: ['CREATE'] # Only do this on CREATE
+
+  ContrailVrouterModuleConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+      - name: phy_int
+      - name: bond_int
+      - name: bond_int_members
+      - name: vlan_parent
+      - name: contrail_repo
+      - name: bond_mode
+      - name: bond_policy
+      config: |
+        #!/bin/bash
+        phy_int=$phy_int
+        bond_int=$bond_int
+        bond_int_members=$bond_int_members
+        bond_mode=$bond_mode
+        bond_policy=$bond_policy
+        vlan_parent=$vlan_parent
+        contrail_repo=$contrail_repo
+        if [[ ${contrail_repo} ]]; then
+          yum install -y contrail-vrouter-utils
+        fi
+        function pkt_setup () {
+            for f in /sys/class/net/$1/queues/rx-*
+            do
+                q="$(echo $f | cut -d '-' -f2)"
+                r=$(($q%32))
+                s=$(($q/32))
+                ((mask=1<<$r))
+                str=(`printf "%x" $mask`)
+                if [ $s -gt 0 ]; then
+                    for ((i=0; i < $s; i++))
+                    do
+                        str+=,00000000
+                    done
+                fi
+                echo $str > $f/rps_cpus
+            done
+            ifconfig $1 up
+        }
+        function insert_vrouter() {
+            if [ -f /sys/class/net/pkt1/queues/rx-0/rps_cpus ]; then
+                pkt_setup pkt1
+            fi
+            if [ -f /sys/class/net/pkt2/queues/rx-0/rps_cpus ]; then
+                pkt_setup pkt2
+            fi
+            if [ -f /sys/class/net/pkt3/queues/rx-0/rps_cpus ]; then
+                pkt_setup pkt3
+            fi
+            DEV_MAC=$(cat /sys/class/net/${phy_int}/address)
+            vif --create vhost0 --mac $DEV_MAC
+            vif --add ${phy_int} --mac $DEV_MAC --vrf 0 --vhost-phys --type physical
+            vif --add vhost0 --mac $DEV_MAC --vrf 0 --type vhost --xconnect ${phy_int}
+            ip link set vhost0 up
+            return 0
+        }
+        if [[ ${bond_int} ]]; then
+           bond_int_member_list=$(echo ${bond_int_members} | tr "," "\n")
+           ip link add name ${bond_int} type bond
+           echo ${bond_mode} > /sys/class/net/${bond_int}/bonding/mode
+           echo ${bond_policy} > /sys/class/net/${bond_int}/bonding/xmit_hash_policy
+           for member in ${bond_int_member_list}; do
+               ip link set dev $member master ${bond_int}
+           done
+        fi
+        if [[ ${vlan_parent} ]]; then
+            vlanId=`echo ${phy_int} | awk -F"vlan" '{print $2}'`
+            ip link add name ${phy_int} link ${vlan_parent} type vlan id ${vlanId}
+        fi
+        if [[ ${contrail_repo} ]]; then
+          yumdownloader contrail-vrouter --destdir /tmp
+          cd /tmp
+          rpm2cpio /tmp/contrail-vrouter*.rpm | cpio -idmv
+          cp `find /tmp/lib/modules -name vrouter.ko |tail -1` /tmp
+          insmod /tmp/vrouter.ko
+        else
+          modprobe vrouter
+        fi
+        insert_vrouter
+        if [[ `ifconfig ${phy_int} |grep "inet "` ]]; then
+          def_gw=''
+          if [[ `ip route show |grep default|grep ${phy_int}` ]]; then
+            def_gw=`ip route show |grep default|grep ${phy_int}|awk '{print $3}'`
+          fi
+          ip=`ifconfig ${phy_int} |grep "inet "|awk '{print $2}'`
+          mask=`ifconfig ${phy_int} |grep "inet "|awk '{print $4}'`
+          ip address delete $ip/$mask dev ${phy_int}
+          ip address add $ip/$mask dev vhost0
+          if [[ $def_gw ]]; then
+            ip route add default via $def_gw
+          fi
+        fi
diff --git a/extraconfig/pre_network/contrail/contrail_dpdk_pre_network.yaml b/extraconfig/pre_network/contrail/contrail_dpdk_pre_network.yaml
new file mode 100644
index 00000000..623eb7e0
--- /dev/null
+++ b/extraconfig/pre_network/contrail/contrail_dpdk_pre_network.yaml
@@ -0,0 +1,168 @@
+heat_template_version: pike
+
+# NOTE: You don't need to pass the parameter explicitly from the
+# parent template, it can be specified via the parameter_defaults
+# in the resource_registry instead, if you want to override the default
+# and/or share values with other templates in the tree.
+parameters:
+  ContrailRepo:
+    type: string
+    default: ''
+  ContrailVrouterPhysicalInterface:
+    default: 'eth0'
+    description: vRouter physical interface
+    type: string
+  ContrailVlanParentInterface:
+    default: ''
+    description: Parent interface of vlan interface
+    type: string
+  ContrailBondInterface:
+    default: ''
+    description: Bond interface name
+    type: string
+  ContrailBondInterfaceMembers:
+    default: ''
+    description: Bond interface members
+    type: string
+  ContrailBondMode:
+    default: '4'
+    description: Bond Mode
+    type: string
+  ContrailBondPolicy:
+    default: '1'
+    description: Bond Policy
+    type: string
+  ContrailDpdkHugePages:
+    default: '2048'
+    description: DPDK Hugepages setting
+    type: string
+  RoleParameters:
+    type: json
+    description: Role Specific parameters
+    default: {}
+  ServiceNames:
+    type: comma_delimited_list
+    default: []
+  server:
+    type: string
+
+description: >
+  This template installs the Contrail dpdk packages in order
+  to bring vhost0 interface up. Vhost0 interface must be up before
+  os-net-config takes over.
+
+resources:
+
+  ContrailVrouterDpdkDeployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      name: ContrailVrouterDpdkDeployment
+      server:  {get_param: server}
+      config: {get_resource: ContrailVrouterDpdkConfig}
+      input_values:
+        phy_int: {get_param: ContrailVrouterPhysicalInterface}
+        bond_int: {get_param: ContrailBondInterface}
+        bond_int_members: {get_param: ContrailBondInterfaceMembers}
+        vlan_parent: {get_param: ContrailVlanParentInterface}
+        contrail_repo: {get_param: ContrailRepo}
+        bond_mode: {get_param: ContrailBondMode}
+        bond_policy: {get_param: ContrailBondPolicy}
+        dpdk_hugepages: {get_param: ContrailDpdkHugePages}
+      actions: ['CREATE'] # Only do this on CREATE
+
+  ContrailVrouterDpdkConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config: |
+        #!/bin/bash
+        phy_int=$phy_int
+        bond_int=$bond_int
+        bond_int_members=$bond_int_members
+        bond_mode=$bond_mode
+        bond_policy=$bond_policy
+        vlan_parent=$vlan_parent
+        contrail_repo=$contrail_repo
+        dpdk_hugepages=$dpdk_hugepages
+        echo "vm.nr_hugepages = $dpdk_hugepages" >> /etc/sysctl.conf
+        echo "vm.max_map_count = 128960" >> /etc/sysctl.conf
+        echo "kernel.core_pattern = /var/crashes/core.%e.%p.%h.%t" >> /etc/sysctl.conf
+        echo "net.ipv4.tcp_keepalive_time = 5" >> /etc/sysctl.conf
+        echo "net.ipv4.tcp_keepalive_probes = 5" >> /etc/sysctl.conf
+        echo "net.ipv4.tcp_keepalive_intvl = 1" >> /etc/sysctl.conf
+        /sbin/sysctl --system
+        modprobe uio
+        if [[ ${contrail_repo} ]]; then
+          yum install -y contrail-vrouter-utils contrail-vrouter-dpdk contrail-vrouter-dpdk-init
+        fi
+        pci_address=`ethtool -i ${phy_int} |grep bus-info| awk '{print $2}' |tr -d ' '`
+        if [[ ${vlan_parent} ]]; then
+           pci_address=`ethtool -i ${vlan_parent} |grep bus-info| awk '{print $2}' |tr -d ' '`
+        fi
+        if [[ ${bond_int} ]]; then
+          bond_int_member_list=$(echo ${bond_int_members} | tr "," "\n")
+          cat <<EOF> /etc/sysconfig/network-scripts/ifcfg-${bond_int}
+        DEVICE=${bond_int}
+        BOOTPROTO=none
+        ONBOOT=yes
+        USERCTL=no
+        BONDING_OPTS="mode=${bond_mode} xmit_hash_policy=${bond_policy}"
+        EOF
+          for member in ${bond_int_member_list}; do
+            cat <<EOF> /etc/sysconfig/network-scripts/ifcfg-${member}
+        DEVICE=${member}
+        BOOTPROTO=none
+        ONBOOT=yes
+        MASTER=${bond_int}
+        SLAVE=yes
+        USERCTL=no
+        EOF
+            ip link set dev ${member} down
+          done
+          ifup ${bond_int}
+          pci_address=0000:00:00.0
+        fi
+        if [[ ${vlan_parent} ]]; then
+          echo ${vlan_parent} >> /tmp/vlan_parent
+          vlanId=`echo ${phy_int} | awk -F"vlan" '{print $2}'`
+          ip link add name ${phy_int} link ${vlan_parent} type vlan id ${vlanId}
+        fi
+        cat <<EOF > /etc/contrail/agent_param
+        LOG=/var/log/contrail.log
+        CONFIG=/etc/contrail/contrail-vrouter-agent.conf
+        prog=/usr/bin/contrail-vrouter-agent
+        pname=contrail-vrouter-agent
+        LIBDIR=/usr/lib64
+        DEVICE=vhost0
+        dev=${phy_int}
+        vgw_subnet_ip=__VGW_SUBNET_IP__
+        vgw_intf=__VGW_INTF_LIST__
+        LOGFILE=--log-file=/var/log/contrail/vrouter.log
+        EOF
+          mac=`ip link sh dev ${phy_int} | grep link/ether|awk '{print $2}' | tr -d ' '`
+          cat <<EOF > /etc/contrail/contrail-vrouter-agent.conf
+        [DEFAULT]
+        platform=dpdk
+        physical_interface_address=$pci_address
+        physical_interface_mac=$mac
+        physical_uio_driver=uio_pci_generic
+        [VIRTUAL-HOST-INTERFACE]
+        physical_interface=${phy_int}
+        name=vhost0
+        EOF
+        echo $pci_address > /etc/contrail/dpdk_pci
+        echo $mac > /etc/contrail/dpdk_mac
+        systemctl start supervisor-vrouter
+        if [[ `ifconfig ${phy_int} |grep "inet "` ]]; then
+          def_gw=''
+          if [[ `ip route show |grep default|grep ${phy_int}` ]]; then
+            def_gw=`ip route show |grep default|grep ${phy_int}|awk '{print $3}'`
+          fi
+          ip=`ifconfig ${phy_int} |grep "inet "|awk '{print $2}'`
+          mask=`ifconfig ${phy_int} |grep "inet "|awk '{print $4}'`
+          ip address delete $ip/$mask dev ${phy_int}
+          ip address add $ip/$mask dev vhost0
+          if [[ $def_gw ]]; then
+            ip route add default via $def_gw
+          fi
+        fi