summaryrefslogtreecommitdiffstats
path: root/VNFs
diff options
context:
space:
mode:
authorYury Kylulin <yury.kylulin@intel.com>2021-06-21 15:55:22 +0300
committerYury Kylulin <yury.kylulin@intel.com>2021-06-21 15:55:22 +0300
commit1c4e666aef045b3aa5eb9999e66b44db00cd47c7 (patch)
treee9af14bf4eee842c132a65cc31d1ccfa744d1f80 /VNFs
parentee88ec7c4efce90527d4e77d50a872fa6a1382f5 (diff)
Add support for multisocket memory reservation
Automatic detection of allowed NUMA nodes for memory allocation added to Kubernetes environment. Signed-off-by: Yury Kylulin <yury.kylulin@intel.com> Change-Id: I4f8b2f64971d43414bbd290f7fcbc6210c1d6d71
Diffstat (limited to 'VNFs')
-rw-r--r--VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml4
-rw-r--r--VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py2
-rw-r--r--VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py48
3 files changed, 38 insertions, 16 deletions
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml b/VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml
index 6d5e8205..9e269f60 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml
@@ -17,12 +17,12 @@ spec:
name: hugepages
resources:
requests:
- hugepages-2Mi: 512Mi
+ hugepages-2Mi: 1Gi
memory: 1Gi
cpu: 8
intel.com/intel_sriov_vfio: '1'
limits:
- hugepages-2Mi: 512Mi
+ hugepages-2Mi: 1Gi
memory: 1Gi
cpu: 8
intel.com/intel_sriov_vfio: '1'
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
index 5d6916a7..f4c89ceb 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
@@ -121,8 +121,6 @@ class RapidGeneratorMachine(RapidMachine):
# Start the generator with the -e option so that the cores don't
# start automatically
super().start_prox('-e')
- if self.vim in ['kubernetes']:
- self.remap_all_cpus()
def set_generator_speed(self, speed):
# The assumption is that we only use task 0 for generating
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
index d0ce8c14..515bea55 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
@@ -50,6 +50,8 @@ class RapidMachine(object):
self.machine_params = machine_params
self.vim = vim
self.cpu_mapping = None
+ self.numa_nodes = [ 0 ]
+ self.socket_mem_mb = '512'
if 'config_file' in self.machine_params.keys():
PROXConfigfile = open (self.machine_params['config_file'], 'r')
PROXConfig = PROXConfigfile.read()
@@ -64,19 +66,19 @@ class RapidMachine(object):
def get_cores(self):
return (self.machine_params['cores'])
- def expand_cpuset(self, cpuset):
- """Expand cpuset provided as comma-separated list of CPU numbers and
- CPU ranges of numbers. For more information please see
+ def expand_list_format(self, list):
+ """Expand cpuset list format provided as comma-separated list of
+ numbers and ranges of numbers. For more information please see
https://man7.org/linux/man-pages/man7/cpuset.7.html
"""
- cpuset_expanded = []
- for cpu in cpuset.split(','):
- if '-' in cpu:
- cpu_range = cpu.split('-')
- cpuset_expanded += range(int(cpu_range[0]), int(cpu_range[1]) + 1)
+ list_expanded = []
+ for num in list.split(','):
+ if '-' in num:
+ num_range = num.split('-')
+ list_expanded += range(int(num_range[0]), int(num_range[1]) + 1)
else:
- cpuset_expanded.append(int(cpu))
- return cpuset_expanded
+ list_expanded.append(int(num))
+ return list_expanded
def read_cpuset(self):
"""Read list of cpus on which we allowed to execute
@@ -84,7 +86,7 @@ class RapidMachine(object):
cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.cpus'
cpuset_cpus = self._client.run_cmd(cmd).decode().rstrip()
RapidLog.debug('{} ({}): Allocated cpuset: {}'.format(self.name, self.ip, cpuset_cpus))
- self.cpu_mapping = self.expand_cpuset(cpuset_cpus)
+ self.cpu_mapping = self.expand_list_format(cpuset_cpus)
RapidLog.debug('{} ({}): Expanded cpuset: {}'.format(self.name, self.ip, self.cpu_mapping))
# Log CPU core mapping for user information
@@ -120,6 +122,25 @@ class RapidMachine(object):
RapidLog.debug('{} ({}): cores {} remapped to {}'.format(self.name, self.ip, self.machine_params['cores'], cpus_remapped))
self.machine_params['cores'] = cpus_remapped
+ def read_cpuset_mems(self):
+ """Read list of NUMA nodes on which we allowed to allocate memory
+ """
+ cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.mems'
+ cpuset_mems = self._client.run_cmd(cmd).decode().rstrip()
+ RapidLog.debug('{} ({}): Allowed NUMA nodes: {}'.format(self.name, self.ip, cpuset_mems))
+ self.numa_nodes = self.expand_list_format(cpuset_mems)
+ RapidLog.debug('{} ({}): Expanded allowed NUMA nodes: {}'.format(self.name, self.ip, self.numa_nodes))
+
+ def get_prox_socket_mem_str(self):
+ socket_mem_str = ''
+ for node in range(self.numa_nodes[-1] + 1):
+ if node in self.numa_nodes:
+ socket_mem_str += self.socket_mem_mb + ','
+ else:
+ socket_mem_str += '0,'
+ socket_mem_str = socket_mem_str[:-1]
+ return socket_mem_str
+
def devbind(self):
# Script to bind the right network interface to the poll mode driver
for index, dp_port in enumerate(self.dp_ports, start = 1):
@@ -141,7 +162,9 @@ class RapidMachine(object):
LuaFile.write('local_ip{}="{}"\n'.format(index, dp_port['ip']))
LuaFile.write('local_hex_ip{}=convertIPToHex(local_ip{})\n'.format(index, index))
if self.vim in ['kubernetes']:
- LuaFile.write("eal=\"--socket-mem=512,0 --file-prefix %s --pci-whitelist %s\"\n" % (self.name, self.machine_params['dp_pci_dev']))
+ socket_mem_str = self.get_prox_socket_mem_str()
+ RapidLog.debug('{} ({}): PROX socket mem str: {}'.format(self.name, self.ip, socket_mem_str))
+ LuaFile.write("eal=\"--socket-mem=%s --file-prefix %s --pci-whitelist %s\"\n" % (socket_mem_str, self.name, self.machine_params['dp_pci_dev']))
else:
LuaFile.write("eal=\"\"\n")
if 'mcore' in self.machine_params.keys():
@@ -173,6 +196,7 @@ class RapidMachine(object):
self.devbind()
if self.vim in ['kubernetes']:
self.read_cpuset()
+ self.read_cpuset_mems()
self.remap_all_cpus()
_, prox_config_file_name = os.path.split(self.machine_params['config_file'])
self.generate_lua()