diff options
Diffstat (limited to 'VNFs/DPPD-PROX/helper-scripts/rapid')
17 files changed, 474 insertions, 80 deletions
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/Dockerfile b/VNFs/DPPD-PROX/helper-scripts/rapid/Dockerfile index 2c25f097..fef0fcaf 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/Dockerfile +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/Dockerfile @@ -17,42 +17,101 @@ ################################################## # Build all components in separate builder image # ################################################## -FROM centos:7 as builder -ARG BUILD_DIR=/opt/rapid +FROM ubuntu:20.04 as builder -COPY ./port_info ${BUILD_DIR}/port_info +ARG DPDK_VERSION=22.07 +ENV DPDK_VERSION=${DPDK_VERSION} -COPY ./deploycentostools.sh ${BUILD_DIR}/ -RUN chmod +x ${BUILD_DIR}/deploycentostools.sh \ - && ${BUILD_DIR}/deploycentostools.sh -k deploy +ARG BUILD_DIR="/opt/rapid" +ENV BUILD_DIR=${BUILD_DIR} + +ENV DEBIAN_FRONTEND=noninteractive + +# Install Dependencies +RUN apt update && apt -y install git wget gcc unzip libpcap-dev libncurses5-dev \ + libedit-dev liblua5.3-dev linux-headers-generic iperf3 pciutils \ + libnuma-dev vim tuna wireshark make driverctl openssh-server sudo \ + meson python3-pyelftools pkg-config + +WORKDIR ${BUILD_DIR} + +# Install DPDK +RUN wget http://fast.dpdk.org/rel/dpdk-${DPDK_VERSION}.tar.xz \ + && tar -xf ./dpdk-${DPDK_VERSION}.tar.xz \ + && cd dpdk-${DPDK_VERSION} \ + && meson build -Dlibdir=lib/x86_64-linux-gnu -Denable_driver_sdk=true \ + && ninja -C build install + +WORKDIR ${BUILD_DIR} + +# Install Prox +RUN git clone https://gerrit.opnfv.org/gerrit/samplevnf \ + && cd samplevnf/VNFs/DPPD-PROX \ + && COMMIT_ID=$(git rev-parse HEAD) \ + && echo "${COMMIT_ID}" > ${BUILD_DIR}/commit_id \ + && meson build \ + && ninja -C build \ + && cp ${BUILD_DIR}/samplevnf/VNFs/DPPD-PROX/build/prox ${BUILD_DIR}/prox + +# Build and copy port info app +WORKDIR ${BUILD_DIR}/samplevnf/VNFs/DPPD-PROX/helper-scripts/rapid/port_info +RUN meson build \ + && ninja -C build \ + && cp ${BUILD_DIR}/samplevnf/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/build/port_info_app ${BUILD_DIR}/port_info_app + +RUN ldconfig && pkg-config --modversion libdpdk > ${BUILD_DIR}/dpdk_version +# Create Minimal Install +RUN ldd ${BUILD_DIR}/prox | awk '$2 ~ /=>/ {print $3}' >> ${BUILD_DIR}/list_of_install_components \ + && echo "${BUILD_DIR}/prox" >> ${BUILD_DIR}/list_of_install_components \ + && echo "${BUILD_DIR}/port_info_app" >> ${BUILD_DIR}/list_of_install_components \ + && echo "${BUILD_DIR}/commit_id" >> ${BUILD_DIR}/list_of_install_components \ + && echo "${BUILD_DIR}/dpdk_version" >> ${BUILD_DIR}/list_of_install_components \ + && find /usr/local/lib/x86_64-linux-gnu -not -path '*/\.*' >> ${BUILD_DIR}/list_of_install_components \ + && tar -czvhf ${BUILD_DIR}/install_components.tgz -T ${BUILD_DIR}/list_of_install_components ############################# # Create slim runtime image # ############################# -FROM centos:7 +FROM ubuntu:20.04 + +ARG BUILD_DIR="/opt/rapid" +ENV BUILD_DIR=${BUILD_DIR} -ARG BUILD_DIR=/opt/rapid +ENV DEBIAN_FRONTEND=noninteractive + +# Install Runtime Dependencies +RUN apt update -y +# Install required dynamically linked libraries + required packages +RUN apt -y install sudo openssh-server libatomic1 -COPY ./deploycentostools.sh ${BUILD_DIR}/ COPY --from=builder ${BUILD_DIR}/install_components.tgz ${BUILD_DIR}/install_components.tgz -COPY --from=builder ${BUILD_DIR}/src ${BUILD_DIR}/src -RUN chmod a+rwx ${BUILD_DIR} && chmod +x ${BUILD_DIR}/deploycentostools.sh \ - && ${BUILD_DIR}/deploycentostools.sh -k runtime_image +WORKDIR / +RUN tar -xvf ${BUILD_DIR}/install_components.tgz --skip-old-files +RUN ldconfig +RUN rm ${BUILD_DIR}/install_components.tgz # Expose SSH and PROX ports EXPOSE 22 8474 +RUN useradd -rm -d /home/rapid -s /bin/bash -g root -G sudo -u 1000 rapid \ + && chmod 777 ${BUILD_DIR} \ + && echo 'rapid:rapid' | chpasswd \ + && mkdir /home/rapid/.ssh + # Copy SSH keys -COPY ./rapid_rsa_key.pub /home/centos/.ssh/authorized_keys +COPY ./rapid_rsa_key.pub /home/rapid/.ssh/authorized_keys COPY ./rapid_rsa_key.pub /root/.ssh/authorized_keys -RUN chown centos:centos /home/centos/.ssh/authorized_keys \ - && chmod 600 /home/centos/.ssh/authorized_keys \ +RUN chown rapid:root /home/rapid/.ssh/authorized_keys \ + && chmod 600 /home/rapid/.ssh/authorized_keys \ && chown root:root /root/.ssh/authorized_keys \ && chmod 600 /root/.ssh/authorized_keys +#RUN apt-get clean && apt autoremove --purge +RUN apt-get autoremove -y && apt-get clean all && rm -rf /var/cache/apt + # Copy startup script COPY ./start.sh /start.sh RUN chmod +x /start.sh diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/esp.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/esp.cfg new file mode 100644 index 00000000..31728daf --- /dev/null +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/esp.cfg @@ -0,0 +1,47 @@ +[lua] +dofile("parameters.lua") + +[eal options] +-n=4 ; force number of memory channels +no-output=no ; disable DPDK debug output +eal=--proc-type auto ${eal} + +[port 0] +name=if0 +mac=hardware +rx desc=2048 +tx desc=2048 +vlan=yes +vdev=esp_tap +local ipv4=$local_ip1 + +[defaults] +mempool size=64K + +[global] +name=${name} + +[core $mcore] +mode=master + +[core $cores] +name=enc +task=0 +mode=esp_enc +sub mode=l3 +remote ipv4=$dest_ip1 +rx port=if0 +tx cores=$altcores task=0 +drop=yes + + +[core $altcores] +name=dec +task=0 +mode=esp_dec +sub mode=l3 +remote ipv4=$dest_ip1 +rx ring=yes +tx port=if0 +drop=yes + diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/helper.lua b/VNFs/DPPD-PROX/helper-scripts/rapid/helper.lua index 1b4c657b..a5633409 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/helper.lua +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/helper.lua @@ -21,7 +21,7 @@ function convertIPToHex(ip) return "IP ADDRESS ERROR" end - local chunks = {ip:match("^(%d+)%.(%d+)%.(%d+)%.(%d+)(\/%d+)$")} + local chunks = {ip:match("^(%d+)%.(%d+)%.(%d+)%.(%d+)(/%d+)$")} if #chunks == 5 then for i,v in ipairs(chunks) do if i < 5 then diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/meson.build b/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/meson.build new file mode 100644 index 00000000..f2efd667 --- /dev/null +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/meson.build @@ -0,0 +1,101 @@ +## +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +project('port-info', 'C', + version: + run_command(['git', 'describe', + '--abbrev=8', '--dirty', '--always']).stdout().strip(), + license: 'Apache', + default_options: ['buildtype=release', 'c_std=gnu99'], + meson_version: '>= 0.47' +) + +cc = meson.get_compiler('c') + +# Configure options for prox +# Grab the DPDK version here "manually" as it is not available in the dpdk_dep +# object +dpdk_version = run_command('pkg-config', '--modversion', 'libdpdk').stdout() + + +cflags = [ + '-DPROGRAM_NAME="port_info_app"', + '-fno-stack-protector', + '-DGRE_TP', + '-D_GNU_SOURCE'] # for PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP + +# Add configured cflags to arguments +foreach arg: cflags + add_project_arguments(arg, language: 'c') +endforeach + +# enable warning flags if they are supported by the compiler +warning_flags = [ + '-Wno-unused', + '-Wno-unused-parameter', + '-Wno-unused-result', + '-Wno-deprecated-declarations'] + +foreach arg: warning_flags + if cc.has_argument(arg) + add_project_arguments(arg, language: 'c') + endif +endforeach + +has_sym_args = [ + [ 'HAVE_LIBEDIT_EL_RFUNC_T', 'histedit.h', + 'el_rfunc_t' ], +] +config = configuration_data() +foreach arg:has_sym_args + config.set(arg[0], cc.has_header_symbol(arg[1], arg[2])) +endforeach +configure_file(output : 'libedit_autoconf.h', configuration : config) + +# All other dependencies +dpdk_dep = dependency('libdpdk', required: true) +tinfo_dep = dependency('tinfo', required: false) +threads_dep = dependency('threads', required: true) +pcap_dep = dependency('pcap', required: true) +libedit_dep = dependency('libedit', required: true) +math_dep = cc.find_library('m', required : false) +dl_dep = cc.find_library('dl', required : true) + +deps = [dpdk_dep, + tinfo_dep, + threads_dep, + pcap_dep, + libedit_dep, + math_dep, + dl_dep] + +# Explicitly add these to the dependency list +deps += [cc.find_library('rte_bus_pci', required: true)] +deps += [cc.find_library('rte_bus_vdev', required: true)] + +if dpdk_version.version_compare('<20.11.0') +deps += [cc.find_library('rte_pmd_ring', required: true)] +else +deps += [cc.find_library('rte_net_ring', required: true)] +endif + +sources = files( + 'port_info.c') + +executable('port_info_app', + sources, + c_args: cflags, + dependencies: deps, + install: true) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/port_info.c b/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/port_info.c index 79bd0c0b..917c0636 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/port_info.c +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/port_info/port_info.c @@ -21,7 +21,11 @@ #include <rte_version.h> static const uint16_t rx_rings = 1, tx_rings = 1; +#if RTE_VERSION < RTE_VERSION_NUM(21,11,0,0) static const struct rte_eth_conf port_conf = { .link_speeds = ETH_LINK_SPEED_AUTONEG }; +#else +static const struct rte_eth_conf port_conf = { .link_speeds = RTE_ETH_LINK_SPEED_AUTONEG }; +#endif static inline int port_info(void) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py index c1aade6b..8754ebc4 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py @@ -47,8 +47,10 @@ class prox_ctrl(object): on %s, attempt: %d" % (self._ip, attempts)) while True: try: - self.run_cmd('test -e /opt/rapid/system_ready_for_rapid') - break + if (self.run_cmd('test -e /opt/rapid/system_ready_for_rapid \ + && echo exists')): + break + time.sleep(2) except RuntimeWarning as ex: RapidLog.debug("RuntimeWarning %d:\n%s" % (ex.returncode, ex.output.strip())) @@ -103,9 +105,12 @@ class prox_ctrl(object): def scp_put(self, src, dst): self._sshclient.scp_put(src, dst) + RapidLog.info("Copying from {} to {}:{}".format(src, self._ip, dst)) def scp_get(self, src, dst): self._sshclient.scp_get('/home/' + self._user + src, dst) + RapidLog.info("Copying from {}:/home/{}{} to {}".format(self._ip, + self._user, src, dst)) class prox_sock(object): def __init__(self, sock): diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py index b56fbe1d..27d2430d 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py @@ -21,7 +21,7 @@ class RapidDefaults(object): Class to define the test defaults """ test_params = { - 'version' : '2021.03.15', # Please do NOT change, used for debugging + 'version' : '2023.01.16', # Please do NOT change, used for debugging 'environment_file' : 'rapid.env', #Default string for environment 'test_file' : 'tests/basicrapid.test', #Default string for test 'machine_map_file' : 'machine.map', #Default string for machine map file diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py index 500be5b4..ea42fc9a 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py @@ -226,7 +226,14 @@ class FlowSizeTest(RapidTest): # the drop rate is below a treshold, either we want that no # packet has been lost during the test. # This can be specified by putting 0 in the .test file - elif ((iteration_data['drop_rate'] < self.test['drop_rate_threshold']) or (iteration_data['abs_dropped']==self.test['drop_rate_threshold']==0)) and (iteration_data['lat_avg']< self.test['lat_avg_threshold']) and (iteration_data['lat_perc']< self.test['lat_perc_threshold']) and (iteration_data['lat_max'] < self.test['lat_max_threshold'] and iteration_data['mis_ordered'] <= self.test['mis_ordered_threshold']): + elif ((self.get_pps(speed,size) - iteration_data['pps_tx']) / self.get_pps(speed,size)) \ + < self.test['generator_threshold'] and \ + ((iteration_data['drop_rate'] < self.test['drop_rate_threshold']) or \ + (iteration_data['abs_dropped']==self.test['drop_rate_threshold']==0)) and \ + (iteration_data['lat_avg']< self.test['lat_avg_threshold']) and \ + (iteration_data['lat_perc']< self.test['lat_perc_threshold']) and \ + (iteration_data['lat_max'] < self.test['lat_max_threshold'] and \ + iteration_data['mis_ordered'] <= self.test['mis_ordered_threshold']): end_data = copy.deepcopy(iteration_data) end_prefix = copy.deepcopy(iteration_prefix) success = True diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py index b0516eeb..de7e6ae3 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py @@ -45,7 +45,7 @@ class IrqTest(RapidTest): max_loop_duration = 0 machine_details = {} for machine in self.machines: - buckets=machine.socket.show_irq_buckets(1) + buckets=machine.socket.show_irq_buckets(machine.get_cores()[0]) if max_loop_duration == 0: # First time we go through the loop, we need to initialize # result_details diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_deployment.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_deployment.py index bfb81611..1d1112f7 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_deployment.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_deployment.py @@ -32,7 +32,7 @@ class K8sDeployment: """ LOG_FILE_NAME = "createrapidk8s.log" SSH_PRIVATE_KEY = "./rapid_rsa_key" - SSH_USER = "centos" + SSH_USER = "rapid" POD_YAML_TEMPLATE_FILE_NAME = "pod-rapid.yaml" @@ -111,7 +111,7 @@ class K8sDeployment: pod_name = self._create_config.get( "POD%d" % i, "name") else: - pod_name = "pod-rapid-%d" % i + pod_name = "prox-pod-%d" % i # Search for POD hostname if self._create_config.has_option("POD%d" % i, @@ -121,6 +121,14 @@ class K8sDeployment: else: pod_nodeselector_hostname = None + # Search for POD spec + if self._create_config.has_option("POD%d" % i, + "spec_file_name"): + pod_spec_file_name = self._create_config.get( + "POD%d" % i, "spec_file_name") + else: + pod_spec_file_name = K8sDeployment.POD_YAML_TEMPLATE_FILE_NAME + # Search for POD dataplane static IP if self._create_config.has_option("POD%d" % i, "dp_ip"): @@ -139,6 +147,7 @@ class K8sDeployment: pod = Pod(pod_name, self._namespace) pod.set_nodeselector(pod_nodeselector_hostname) + pod.set_spec_file_name(pod_spec_file_name) pod.set_dp_ip(pod_dp_ip) pod.set_dp_subnet(pod_dp_subnet) pod.set_id(i) @@ -157,7 +166,7 @@ class K8sDeployment: # Create PODs using template from yaml file for pod in self._pods: self._log.info("Creating POD %s...", pod.get_name()) - pod.create_from_yaml(K8sDeployment.POD_YAML_TEMPLATE_FILE_NAME) + pod.create_from_yaml() # Wait for PODs to start for pod in self._pods: @@ -167,6 +176,7 @@ class K8sDeployment: for pod in self._pods: pod.set_ssh_credentials(K8sDeployment.SSH_USER, K8sDeployment.SSH_PRIVATE_KEY) pod.get_sriov_dev_mac() + pod.get_qat_dev() def save_runtime_config(self, config_file_name): self._log.info("Saving config %s for runrapid script...", @@ -203,6 +213,10 @@ class K8sDeployment: "dp_mac1", pod.get_dp_mac()) self._runtime_config.set("M%d" % pod.get_id(), "dp_pci_dev", pod.get_dp_pci_dev()) + if (pod.get_qat_pci_dev()): + for qat_index, qat_device in enumerate(pod.get_qat_pci_dev()): + self._runtime_config.set("M%d" % pod.get_id(), + "qat_pci_dev%d" % qat_index, qat_device) self._runtime_config.set("M%d" % pod.get_id(), "dp_ip1", pod.get_dp_ip() + "/" + pod.get_dp_subnet()) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_pod.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_pod.py index d15fe7ff..beaedd69 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_pod.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_k8s_pod.py @@ -32,6 +32,7 @@ class Pod: _name = "pod" _namespace = "default" _nodeSelector_hostname = None + _spec_filename = None _last_status = None _id = None _admin_ip = None @@ -49,6 +50,7 @@ class Pod: self._name = name self._namespace = namespace self._ssh_client = SSHClient(logger_name = logger_name) + self.qat_vf = [] def __del__(self): """Destroy POD. Do a cleanup. @@ -56,10 +58,11 @@ class Pod: if self._ssh_client is not None: self._ssh_client.disconnect() - def create_from_yaml(self, file_name): + def create_from_yaml(self): """Load POD description from yaml file. """ - with open(path.join(path.dirname(__file__), file_name)) as yaml_file: + with open(path.join(path.dirname(__file__), + self._spec_filename)) as yaml_file: self.body = yaml.safe_load(yaml_file) self.body["metadata"]["name"] = self._name @@ -67,14 +70,16 @@ class Pod: if (self._nodeSelector_hostname is not None): if ("nodeSelector" not in self.body["spec"]): self.body["spec"]["nodeSelector"] = {} - self.body["spec"]["nodeSelector"]["kubernetes.io/hostname"] = self._nodeSelector_hostname + self.body["spec"]["nodeSelector"]["kubernetes.io/hostname"] = \ + self._nodeSelector_hostname self._log.debug("Creating POD, body:\n%s" % self.body) try: self.k8s_CoreV1Api.create_namespaced_pod(body = self.body, namespace = self._namespace) except client.rest.ApiException as e: - self._log.error("Couldn't create POD %s!\n%s\n" % (self._name, e)) + self._log.error("Couldn't create POD %s!\n%s\n" % (self._name, + e)) def terminate(self): """Terminate POD. Close SSH connection. @@ -138,6 +143,9 @@ class Pod: def get_dp_pci_dev(self): return self._sriov_vf + def get_qat_pci_dev(self): + return self.qat_vf + def get_id(self): return self._id @@ -153,6 +161,28 @@ class Pod: self._last_status = pod.status.phase return self._last_status + def get_qat_dev(self): + """Get qat devices if any, assigned by k8s QAT device plugin. + """ + self._log.info("Checking assigned QAT VF for POD %s" % self._name) + ret = self._ssh_client.run_cmd("cat /opt/rapid/k8s_qat_device_plugin_envs") + if ret != 0: + self._log.error("Failed to check assigned QAT VF!" + "Error %s" % self._ssh_client.get_error()) + return -1 + + cmd_output = self._ssh_client.get_output().decode("utf-8").rstrip() + + if cmd_output: + self._log.debug("Before: Using QAT VF %s" % self.qat_vf) + self._log.debug("Environment variable %s" % cmd_output) + for line in cmd_output.splitlines(): + self.qat_vf.append(line.split("=")[1]) + self._log.debug("Using QAT VF %s" % self.qat_vf) + else: + self._log.debug("No QAT devices for this pod") + self.qat_vf = None + def get_sriov_dev_mac(self): """Get assigned by k8s SRIOV network device plugin SRIOV VF devices. Return 0 in case of sucessfull configuration. @@ -173,8 +203,24 @@ class Pod: self._sriov_vf = cmd_output.split(",")[0] self._log.debug("Using first SRIOV VF %s" % self._sriov_vf) - self._log.info("Getting MAC address for assigned SRIOV VF %s" % self._sriov_vf) - self._ssh_client.run_cmd("sudo /opt/rapid/port_info_app -n 4 -w %s" % self._sriov_vf) + # find DPDK version + self._log.info("Checking DPDK version for POD %s" % self._name) + ret = self._ssh_client.run_cmd("cat /opt/rapid/dpdk_version") + if ret != 0: + self._log.error("Failed to check DPDK version" + "Error %s" % self._ssh_client.get_error()) + return -1 + dpdk_version = self._ssh_client.get_output().decode("utf-8").rstrip() + self._log.debug("DPDK version %s" % dpdk_version) + if (dpdk_version >= '20.11.0'): + allow_parameter = 'allow' + else: + allow_parameter = 'pci-whitelist' + + self._log.info("Getting MAC address for assigned SRIOV VF %s" % \ + self._sriov_vf) + self._ssh_client.run_cmd("sudo /opt/rapid/port_info_app -n 4 \ + --{} {}".format(allow_parameter, self._sriov_vf)) if ret != 0: self._log.error("Failed to get MAC address!" "Error %s" % self._ssh_client.get_error()) @@ -204,6 +250,11 @@ class Pod: """ self._nodeSelector_hostname = hostname + def set_spec_file_name(self, file_name): + """Set pod spec filename. + """ + self._spec_filename = file_name + def set_ssh_credentials(self, user, rsa_private_key): """Set SSH credentials for the SSH connection to the POD. """ diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py index e46e46ef..47f858d0 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py @@ -20,6 +20,7 @@ from rapid_log import RapidLog from prox_ctrl import prox_ctrl import os import re +import uuid class RapidMachine(object): """ @@ -41,8 +42,11 @@ class RapidMachine(object): while True: ip_key = 'dp_ip{}'.format(index) mac_key = 'dp_mac{}'.format(index) - if ip_key in machine_params.keys() and mac_key in machine_params.keys(): - dp_port = {'ip': machine_params[ip_key], 'mac' : machine_params[mac_key]} + if ip_key in machine_params.keys(): + if mac_key in machine_params.keys(): + dp_port = {'ip': machine_params[ip_key], 'mac' : machine_params[mac_key]} + else: + dp_port = {'ip': machine_params[ip_key], 'mac' : None} self.dp_ports.append(dict(dp_port)) self.dpdk_port_index.append(index - 1) index += 1 @@ -51,8 +55,6 @@ class RapidMachine(object): self.machine_params = machine_params self.vim = vim self.cpu_mapping = None - self.numa_nodes = [ 0 ] - self.socket_mem_mb = '512' if 'config_file' in self.machine_params.keys(): PROXConfigfile = open (self.machine_params['config_file'], 'r') PROXConfig = PROXConfigfile.read() @@ -79,7 +81,17 @@ class RapidMachine(object): def read_cpuset(self): """Read list of cpus on which we allowed to execute """ - cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.cpus' + cpu_set_file = '/sys/fs/cgroup/cpuset.cpus' + cmd = 'test -e {0} && echo exists'.format(cpu_set_file) + if (self._client.run_cmd(cmd).decode().rstrip()): + cmd = 'cat {}'.format(cpu_set_file) + else: + cpu_set_file = '/sys/fs/cgroup/cpuset/cpuset.cpus' + cmd = 'test -e {0} && echo exists'.format(cpu_set_file) + if (self._client.run_cmd(cmd).decode().rstrip()): + cmd = 'cat {}'.format(cpu_set_file) + else: + RapidLog.critical('{Cannot determine cpuset') cpuset_cpus = self._client.run_cmd(cmd).decode().rstrip() RapidLog.debug('{} ({}): Allocated cpuset: {}'.format(self.name, self.ip, cpuset_cpus)) self.cpu_mapping = self.expand_list_format(cpuset_cpus) @@ -118,24 +130,10 @@ class RapidMachine(object): RapidLog.debug('{} ({}): cores {} remapped to {}'.format(self.name, self.ip, self.machine_params['cores'], cpus_remapped)) self.machine_params['cores'] = cpus_remapped - def read_cpuset_mems(self): - """Read list of NUMA nodes on which we allowed to allocate memory - """ - cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.mems' - cpuset_mems = self._client.run_cmd(cmd).decode().rstrip() - RapidLog.debug('{} ({}): Allowed NUMA nodes: {}'.format(self.name, self.ip, cpuset_mems)) - self.numa_nodes = self.expand_list_format(cpuset_mems) - RapidLog.debug('{} ({}): Expanded allowed NUMA nodes: {}'.format(self.name, self.ip, self.numa_nodes)) - - def get_prox_socket_mem_str(self): - socket_mem_str = '' - for node in range(self.numa_nodes[-1] + 1): - if node in self.numa_nodes: - socket_mem_str += self.socket_mem_mb + ',' - else: - socket_mem_str += '0,' - socket_mem_str = socket_mem_str[:-1] - return socket_mem_str + if 'altcores' in self.machine_params.keys(): + cpus_remapped = self.remap_cpus(self.machine_params['altcores']) + RapidLog.debug('{} ({}): altcores {} remapped to {}'.format(self.name, self.ip, self.machine_params['altcores'], cpus_remapped)) + self.machine_params['altcores'] = cpus_remapped def devbind(self): # Script to bind the right network interface to the poll mode driver @@ -158,22 +156,47 @@ class RapidMachine(object): LuaFile.write('local_ip{}="{}"\n'.format(index, dp_port['ip'])) LuaFile.write('local_hex_ip{}=convertIPToHex(local_ip{})\n'.format(index, index)) if self.vim in ['kubernetes']: - socket_mem_str = self.get_prox_socket_mem_str() - RapidLog.debug('{} ({}): PROX socket mem str: {}'.format(self.name, self.ip, socket_mem_str)) - LuaFile.write("eal=\"--socket-mem=%s --file-prefix %s --pci-whitelist %s\"\n" % (socket_mem_str, self.name, self.machine_params['dp_pci_dev'])) + cmd = 'cat /opt/rapid/dpdk_version' + dpdk_version = self._client.run_cmd(cmd).decode().rstrip() + if (dpdk_version >= '20.11.0'): + allow_parameter = 'allow' + else: + allow_parameter = 'pci-whitelist' + eal_line = 'eal=\"--file-prefix {}{} --{} {} --force-max-simd-bitwidth=512'.format( + self.name, str(uuid.uuid4()), allow_parameter, + self.machine_params['dp_pci_dev']) + looking_for_qat = True + index = 0 + while (looking_for_qat): + if 'qat_pci_dev{}'.format(index) in self.machine_params: + eal_line += ' --{} {}'.format(allow_parameter, + self.machine_params['qat_pci_dev{}'.format(index)]) + index += 1 + else: + looking_for_qat = False + eal_line += '"\n' + LuaFile.write(eal_line) else: LuaFile.write("eal=\"\"\n") if 'mcore' in self.machine_params.keys(): - LuaFile.write('mcore="%s"\n'% ','.join(map(str, self.machine_params['mcore']))) + LuaFile.write('mcore="%s"\n'% ','.join(map(str, + self.machine_params['mcore']))) if 'cores' in self.machine_params.keys(): - LuaFile.write('cores="%s"\n'% ','.join(map(str, self.machine_params['cores']))) + LuaFile.write('cores="%s"\n'% ','.join(map(str, + self.machine_params['cores']))) + if 'altcores' in self.machine_params.keys(): + LuaFile.write('altcores="%s"\n'% ','.join(map(str, + self.machine_params['altcores']))) if 'ports' in self.machine_params.keys(): - LuaFile.write('ports="%s"\n'% ','.join(map(str, self.machine_params['ports']))) + LuaFile.write('ports="%s"\n'% ','.join(map(str, + self.machine_params['ports']))) if 'dest_ports' in self.machine_params.keys(): for index, dest_port in enumerate(self.machine_params['dest_ports'], start = 1): LuaFile.write('dest_ip{}="{}"\n'.format(index, dest_port['ip'])) LuaFile.write('dest_hex_ip{}=convertIPToHex(dest_ip{})\n'.format(index, index)) - LuaFile.write('dest_hex_mac{}="{}"\n'.format(index , dest_port['mac'].replace(':',' '))) + if dest_port['mac']: + LuaFile.write('dest_hex_mac{}="{}"\n'.format(index , + dest_port['mac'].replace(':',' '))) if 'gw_vm' in self.machine_params.keys(): for index, gw_ip in enumerate(self.machine_params['gw_ips'], start = 1): @@ -193,7 +216,6 @@ class RapidMachine(object): self.devbind() if self.vim in ['kubernetes']: self.read_cpuset() - self.read_cpuset_mems() self.remap_all_cpus() _, prox_config_file_name = os.path.split(self. machine_params['config_file']) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py index 445aed4b..143323b8 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py @@ -63,12 +63,6 @@ class RapidConfigParser(object): test_params['user'] = config.get('ssh', 'user') if config.has_option('ssh', 'key'): test_params['key'] = config.get('ssh', 'key') - if test_params['user'] in ['rapid']: - if test_params['key'] != 'rapid_rsa_key': - RapidLog.debug(("Key file {} for user {} overruled by key file:" - " rapid_rsa_key").format(test_params['key'], - test_params['user'])) - test_params['key'] = 'rapid_rsa_key' else: test_params['key'] = None if config.has_option('ssh', 'password'): @@ -91,7 +85,7 @@ class RapidConfigParser(object): 'flowsize','warmupflowsize','warmuptime', 'steps']: test[option] = int(testconfig.get(section, option)) elif option in ['startspeed', 'step', 'drop_rate_threshold', - 'lat_avg_threshold','lat_perc_threshold', + 'generator_threshold','lat_avg_threshold','lat_perc_threshold', 'lat_max_threshold','accuracy','maxr','maxz', 'ramp_step','warmupspeed','mis_ordered_threshold']: test[option] = float(testconfig.get(section, option)) @@ -99,11 +93,12 @@ class RapidConfigParser(object): test[option] = testconfig.get(section, option) tests.append(dict(test)) for test in tests: - if test['test'] in ['flowsizetest','TST009test']: + if test['test'] in ['flowsizetest', 'TST009test', 'increment_till_fail']: if 'drop_rate_threshold' not in test.keys(): test['drop_rate_threshold'] = 0 - latency_thresholds = ['lat_avg_threshold','lat_perc_threshold','lat_max_threshold','mis_ordered_threshold'] - for threshold in latency_thresholds: + thresholds = ['generator_threshold','lat_avg_threshold', \ + 'lat_perc_threshold','lat_max_threshold','mis_ordered_threshold'] + for threshold in thresholds: if threshold not in test.keys(): test[threshold] = inf test_params['tests'] = tests @@ -142,7 +137,8 @@ class RapidConfigParser(object): for option in options: if option in ['prox_socket','prox_launch_exit','monitor']: machine[option] = testconfig.getboolean(section, option) - elif option in ['mcore', 'cores', 'gencores','latcores']: + elif option in ['mcore', 'cores', 'gencores', 'latcores', + 'altcores']: machine[option] = ast.literal_eval(testconfig.get( section, option)) elif option in ['bucket_size_exp']: @@ -169,10 +165,13 @@ class RapidConfigParser(object): while True: dp_ip_key = 'dp_ip{}'.format(index) dp_mac_key = 'dp_mac{}'.format(index) - if dp_ip_key in machines[int(machine['dest_vm'])-1].keys() and \ - dp_mac_key in machines[int(machine['dest_vm'])-1].keys(): - dp_port = {'ip': machines[int(machine['dest_vm'])-1][dp_ip_key], - 'mac' : machines[int(machine['dest_vm'])-1][dp_mac_key]} + if dp_ip_key in machines[int(machine['dest_vm'])-1].keys(): + if dp_mac_key in machines[int(machine['dest_vm'])-1].keys(): + dp_port = {'ip': machines[int(machine['dest_vm'])-1][dp_ip_key], + 'mac' : machines[int(machine['dest_vm'])-1][dp_mac_key]} + else: + dp_port = {'ip': machines[int(machine['dest_vm'])-1][dp_ip_key], + 'mac' : None} dp_ports.append(dict(dp_port)) index += 1 else: diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py index 6badfb45..deba695f 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py @@ -261,6 +261,7 @@ class RapidTest(object): 'lat_perc' : '', 'lat_max' : '', 'abs_drop_rate' : '', + 'mis_ordered' : '', 'drop_rate' : ''} RapidLog.info(self.report_result(flow_number, size, iteration_data, iteration_prefix )) @@ -346,6 +347,7 @@ class RapidTest(object): 'lat_perc' : '', 'lat_max' : '', 'abs_drop_rate' : '', + 'mis_ordered' : '', 'drop_rate' : ''} RapidLog.info(self.report_result(flow_number, size, time_loop_data, time_loop_prefix)) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py index 7a1c8eb5..7ec270a1 100755 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py @@ -59,12 +59,20 @@ class RapidTestManager(object): def run_tests(self, test_params): test_params = RapidConfigParser.parse_config(test_params) - RapidLog.debug(test_params) monitor_gen = monitor_sut = False background_machines = [] sut_machine = gen_machine = None configonly = test_params['configonly'] + machine_names = [] + machine_counter = {} for machine_params in test_params['machines']: + if machine_params['name'] not in machine_names: + machine_names.append(machine_params['name']) + machine_counter[machine_params['name']] = 1 + else: + machine_counter[machine_params['name']] += 1 + machine_params['name'] = '{}_{}'.format(machine_params['name'], + machine_counter[machine_params['name']]) if 'gencores' in machine_params.keys(): machine = RapidGeneratorMachine(test_params['key'], test_params['user'], test_params['password'], @@ -94,6 +102,7 @@ class RapidTestManager(object): if machine_params['prox_socket']: sut_machine = machine self.machines.append(machine) + RapidLog.debug(test_params) try: prox_executor = concurrent.futures.ThreadPoolExecutor(max_workers=len(self.machines)) self.future_to_prox = {prox_executor.submit(machine.start_prox): machine for machine in self.machines} diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/start.sh b/VNFs/DPPD-PROX/helper-scripts/rapid/start.sh index 7fbeedf8..78772dd2 100755 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/start.sh +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/start.sh @@ -17,7 +17,8 @@ function save_k8s_envs() { - printenv | grep "PCIDEVICE_INTEL_COM" > /opt/rapid/k8s_sriov_device_plugin_envs + printenv | grep "PCIDEVICE" > /opt/rapid/k8s_sriov_device_plugin_envs + printenv | grep "QAT[0-9]" > /opt/rapid/k8s_qat_device_plugin_envs } function create_tun() @@ -34,6 +35,9 @@ create_tun touch /opt/rapid/system_ready_for_rapid # Start SSH server in background -/usr/sbin/sshd +echo "mkdir -p /var/run/sshd" >> /etc/rc.local +service ssh start -exec sleep infinity +echo "rapid ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +sleep infinity diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/encrypt.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/encrypt.test new file mode 100644 index 00000000..bc5e96b8 --- /dev/null +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/encrypt.test @@ -0,0 +1,70 @@ +## +## Copyright (c) 2023 luc.provoost@gmail.com +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +# CHECK README IN THIS DIRECTORY FOR MORE EXPLANATION +# ON PARAMETERS IN THIS FILE + +[TestParameters] +name = EncryptionDecryption +number_of_tests = 1 +total_number_of_test_machines = 2 +lat_percentile = 99 + +[TestM1] +name = Generator +config_file = configs/gen.cfg +dest_vm = 2 +mcore = [0] +gencores = [1] +latcores = [3] +bucket_size_exp = 16 +#prox_launch_exit = false + +[TestM2] +name = Encrypt +config_file = configs/esp.cfg +dest_vm = 1 +mcore = [0] +cores = [1] +altcores=[2] +#prox_socket = true +#prox_launch_exit = false + +[test1] +test=flowsizetest +warmupflowsize=512 +warmupimix=[64] +warmupspeed=1 +warmuptime=2 +# Each element in the imix list will result in a separate test. Each element +# is on its turn a list of packet sizes which will be used during one test +# execution. If you only want to test 1 size, define a list with only one +# element. +#imixs=[[64],[64,250,800,800]] +imixs=[[1500],[512],[256],[128]] +# the number of flows in the list need to be powers of 2, max 2^30 +# If not a power of 2, we will use the lowest power of 2 that is larger than +# the requested number of flows. e.g. 9 will result in 16 flows +flows=[64] +# Setting one of the following thresholds to infinity (inf) +# results in the criterion not being evaluated to rate the test as succesful +drop_rate_threshold = 0.5 +lat_avg_threshold = inf +lat_perc_threshold = inf +lat_max_threshold = inf +accuracy = 5 +startspeed = 250 +#ramp_step = 1 + |