Increase size of connection tracking table

During high load, the default limit of the kernel connection tracking
table (65536) is often too low, resuling in error messages such as:

kernel: nf_conntrack: table full, dropping packet

This patch increases the limit to 500,000.

Since the nf_conntrack kernel module is not always loaded by default, it also
adds a mechanism to load kernel modules via hieradata using the kmod puppet
module. In order to express the needed dependency in puppet that kernel modules
are loaded before sysctl settings are applied, the Exec resources tagged with
'kmod::load' are specified in a resource collector to express that that Exec
resources with the tag should run before Sysctl resources.

Depends-On: I59cc2280ebae315af38fb5008e6ee0073195ae51
Change-Id: Iffa0a77852729786b69945c1e72bc90ad57ce3bb

7 files changed, 19 insertions, 0 deletions

diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index b4b51abf..52b23513 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -14,6 +14,9 @@ nova::network::neutron::dhcp_domain: ''
 
 neutron::allow_overlapping_ips: true
 
+kernel_modules:
+  nf_conntrack: {}
+
 sysctl_settings:
   net.ipv4.tcp_keepalive_intvl:
     value: 1
@@ -21,6 +24,10 @@ sysctl_settings:
     value: 5
   net.ipv4.tcp_keepalive_time:
     value: 5
+  net.nf_conntrack_max:
+    value: 500000
+  net.netfilter.nf_conntrack_max:
+    value: 500000
 
 nova::rabbit_heartbeat_timeout_threshold: 60
 neutron::rabbit_heartbeat_timeout_threshold: 60
diff --git a/puppet/manifests/overcloud_cephstorage.pp b/puppet/manifests/overcloud_cephstorage.pp
index 1d801adc..0db5b45a 100644
--- a/puppet/manifests/overcloud_cephstorage.pp
+++ b/puppet/manifests/overcloud_cephstorage.pp
@@ -16,7 +16,9 @@
 include ::tripleo::packages
 include ::tripleo::firewall
 
+create_resources(kmod::load, hiera('kernel_modules'), {})
 create_resources(sysctl::value, hiera('sysctl_settings'), {})
+Exec <| tag == 'kmod::load' |>  -> Sysctl <| |>
 
 if count(hiera('ntp::servers')) > 0 {
   include ::ntp
diff --git a/puppet/manifests/overcloud_compute.pp b/puppet/manifests/overcloud_compute.pp
index 593cc500..6403a734 100644
--- a/puppet/manifests/overcloud_compute.pp
+++ b/puppet/manifests/overcloud_compute.pp
@@ -16,7 +16,9 @@
 include ::tripleo::packages
 include ::tripleo::firewall
 
+create_resources(kmod::load, hiera('kernel_modules'), {})
 create_resources(sysctl::value, hiera('sysctl_settings'), {})
+Exec <| tag == 'kmod::load' |>  -> Sysctl <| |>
 
 if count(hiera('ntp::servers')) > 0 {
   include ::ntp
diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp
index 70aebddd..ae1f0b9f 100644
--- a/puppet/manifests/overcloud_controller.pp
+++ b/puppet/manifests/overcloud_controller.pp
@@ -20,7 +20,9 @@ $enable_load_balancer = hiera('enable_load_balancer', true)
 
 if hiera('step') >= 1 {
 
+  create_resources(kmod::load, hiera('kernel_modules'), {})
   create_resources(sysctl::value, hiera('sysctl_settings'), {})
+  Exec <| tag == 'kmod::load' |>  -> Sysctl <| |>
 
   $controller_node_ips = split(hiera('controller_node_ips'), ',')
 
diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index e0286162..aae4e2d0 100644
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -39,7 +39,9 @@ $non_pcmk_start = hiera('step') >= 4
 
 if hiera('step') >= 1 {
 
+  create_resources(kmod::load, hiera('kernel_modules'), {})
   create_resources(sysctl::value, hiera('sysctl_settings'), {})
+  Exec <| tag == 'kmod::load' |>  -> Sysctl <| |>
 
   include ::timezone
 
diff --git a/puppet/manifests/overcloud_object.pp b/puppet/manifests/overcloud_object.pp
index 63ac396e..1ac66904 100644
--- a/puppet/manifests/overcloud_object.pp
+++ b/puppet/manifests/overcloud_object.pp
@@ -16,7 +16,9 @@
 include ::tripleo::packages
 include ::tripleo::firewall
 
+create_resources(kmod::load, hiera('kernel_modules'), {})
 create_resources(sysctl::value, hiera('sysctl_settings'), {})
+Exec <| tag == 'kmod::load' |>  -> Sysctl <| |>
 
 if count(hiera('ntp::servers')) > 0 {
   include ::ntp
diff --git a/puppet/manifests/overcloud_volume.pp b/puppet/manifests/overcloud_volume.pp
index 5a69725a..72cd36c3 100644
--- a/puppet/manifests/overcloud_volume.pp
+++ b/puppet/manifests/overcloud_volume.pp
@@ -16,7 +16,9 @@
 include ::tripleo::packages
 include ::tripleo::firewall
 
+create_resources(kmod::load, hiera('kernel_modules'), {})
 create_resources(sysctl::value, hiera('sysctl_settings'), {})
+Exec <| tag == 'kmod::load' |>  -> Sysctl <| |>
 
 if count(hiera('ntp::servers')) > 0 {
   include ::ntp