WIP: Reference Definitions.

This patch adds reference definitions, and dockerfile for
test-containers.

Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>
Change-Id: Ifbd90f80cb1bc91fa0bf6053fc6405891d645412

13 files changed, 540 insertions, 0 deletions

diff --git a/tools/k8s/reference-definitions/network-attachments/ovsdpdk/userspace-ovs-netAttach.yaml b/tools/k8s/reference-definitions/network-attachments/ovsdpdk/userspace-ovs-netAttach.yaml
new file mode 100644
index 00000000..b2ac5c7c
--- /dev/null
+++ b/tools/k8s/reference-definitions/network-attachments/ovsdpdk/userspace-ovs-netAttach.yaml
@@ -0,0 +1,32 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: userspace-ovs-net
+spec:
+  config: '{
+        "cniVersion": "0.3.1",
+        "type": "userspace",
+        "name": "userspace-ovs-net-1",
+        "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
+        "logFile": "/var/log/userspace-ovs-net.log",
+        "logLevel": "debug",
+        "host": {
+                "engine": "ovs-dpdk",
+                "iftype": "vhostuser",
+                "netType": "bridge",
+                "vhost": {
+                        "mode": "server"
+                },
+                "bridge": {
+                        "bridgeName": "vsperf-br0"
+                }
+        },
+        "container": {
+                "engine": "ovs-dpdk",
+                "iftype": "vhostuser",
+                "netType": "interface",
+                "vhost": {
+                        "mode": "client"
+                }
+        }
+    }'
diff --git a/tools/k8s/reference-definitions/network-attachments/sriov/attach-order.sh b/tools/k8s/reference-definitions/network-attachments/sriov/attach-order.sh
new file mode 100755
index 00000000..9ffac806
--- /dev/null
+++ b/tools/k8s/reference-definitions/network-attachments/sriov/attach-order.sh
@@ -0,0 +1,4 @@
+kubectl create -f netAttach-sriov-dpdk-a.yaml
+kubectl create -f netAttach-sriov-dpdk-b.yaml
+kubectl create -f configMap.yaml
+kubectl create -f sriovdp-daemonset.yaml
diff --git a/tools/k8s/reference-definitions/network-attachments/sriov/configMap.yaml b/tools/k8s/reference-definitions/network-attachments/sriov/configMap.yaml
new file mode 100644
index 00000000..53461c0f
--- /dev/null
+++ b/tools/k8s/reference-definitions/network-attachments/sriov/configMap.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sriovdp-config
+  namespace: kube-system
+data:
+  config.json: |
+    {
+        "resourceList": [{
+                "resourceName": "intel_sriov_dpdk_a",
+                "selectors": {
+                    "vendors": ["8086"],
+                    "devices": ["10ed"],
+                    "drivers": ["vfio-pci"],
+                    "pfNames": ["eno3"]
+                }
+            },
+            {
+                "resourceName": "intel_sriov_dpdk_b",
+                "selectors": {
+                    "vendors": ["8086"],
+                    "devices": ["10ed"],
+                    "drivers": ["vfio-pci"],
+                    "pfNames": ["eno4"]
+                }
+            }
+
+        ]
+    }
+
diff --git a/tools/k8s/reference-definitions/network-attachments/sriov/netAttach-sriov-dpdk-b.yaml b/tools/k8s/reference-definitions/network-attachments/sriov/netAttach-sriov-dpdk-b.yaml
new file mode 100644
index 00000000..c876d76d
--- /dev/null
+++ b/tools/k8s/reference-definitions/network-attachments/sriov/netAttach-sriov-dpdk-b.yaml
@@ -0,0 +1,14 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-b
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_dpdk_b
+spec:
+  config: '{
+  "type": "sriov",
+  "cniVersion": "0.3.1",
+  "name": "sriov-network-b",
+  "spoofchk": "off",
+  "trust": "on"
+}'
diff --git a/tools/k8s/reference-definitions/network-attachments/sriov/netAttach-sriov-dpdk.yaml b/tools/k8s/reference-definitions/network-attachments/sriov/netAttach-sriov-dpdk.yaml
new file mode 100644
index 00000000..9678ae4a
--- /dev/null
+++ b/tools/k8s/reference-definitions/network-attachments/sriov/netAttach-sriov-dpdk.yaml
@@ -0,0 +1,14 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_dpdk_a
+spec:
+  config: '{
+  "type": "sriov",
+  "cniVersion": "0.3.1",
+  "name": "sriov-network-a",
+  "spoofchk": "off",
+  "trust": "on"
+}'
diff --git a/tools/k8s/reference-definitions/network-attachments/sriov/sriovdp-daemonset.yaml b/tools/k8s/reference-definitions/network-attachments/sriov/sriovdp-daemonset.yaml
new file mode 100644
index 00000000..91a68f5b
--- /dev/null
+++ b/tools/k8s/reference-definitions/network-attachments/sriov/sriovdp-daemonset.yaml
@@ -0,0 +1,66 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sriov-device-plugin
+  namespace: kube-system
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-sriov-device-plugin-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: sriovdp
+spec:
+  selector:
+    matchLabels:
+      name: sriov-device-plugin
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: sriovdp
+        name: sriov-device-plugin
+    spec:
+      hostNetwork: true
+      hostPID: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: sriov-device-plugin
+      containers:
+      - name: kube-sriovdp
+        image: nfvpe/sriov-device-plugin
+        imagePullPolicy: Never
+        args:
+        - --log-dir=sriovdp
+        - --log-level=10
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: devicesock
+          mountPath: /var/lib/kubelet/
+          readOnly: false
+        - name: log
+          mountPath: /var/log
+        - name: config-volume
+          mountPath: /etc/pcidp
+      volumes:
+        - name: devicesock
+          hostPath:
+            path: /var/lib/kubelet/
+        - name: log
+          hostPath:
+            path: /var/log
+        - name: config-volume
+          configMap:
+            name: sriovdp-config
+            items:
+            - key: config.json
+              path: config.json
diff --git a/tools/k8s/reference-definitions/network-attachments/vpp/userspace-vpp-netAttach-memif.yaml b/tools/k8s/reference-definitions/network-attachments/vpp/userspace-vpp-netAttach-memif.yaml
new file mode 100644
index 00000000..1ac6c245
--- /dev/null
+++ b/tools/k8s/reference-definitions/network-attachments/vpp/userspace-vpp-netAttach-memif.yaml
@@ -0,0 +1,32 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: userspace-vpp-net
+spec:
+  config: '{
+        "cniVersion": "0.3.1",
+        "type": "userspace",
+        "name": "userspace-vpp-net",
+        "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
+        "logFile": "/var/log/userspace-vpp-net-1-cni.log",
+        "logLevel": "debug",
+        "host": {
+                "engine": "vpp",
+                "iftype": "memif",
+                "netType": "interface",
+                "memif": {
+                        "role": "master",
+                        "mode": "ethernet"
+                }
+        },
+        "container": {
+                "engine": "vpp",
+                "iftype": "memif",
+                "netType": "interface",
+                "memif": {
+                        "role": "slave",
+                        "mode": "ethernet"
+                }
+        }
+    }'
+
diff --git a/tools/k8s/reference-definitions/pod-defs/ovsdpdk/userspace-ovs-netapp-pod.yaml b/tools/k8s/reference-definitions/pod-defs/ovsdpdk/userspace-ovs-netapp-pod.yaml
new file mode 100644
index 00000000..64f4a5cb
--- /dev/null
+++ b/tools/k8s/reference-definitions/pod-defs/ovsdpdk/userspace-ovs-netapp-pod.yaml
@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: userspace-ovs-pod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: userspace-ovs-net, userspace-ovs-net
+spec:
+  containers:
+  - name: multi-vhost
+    image: dpdk-app-centos:latest
+    imagePullPolicy: Never
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/podnetinfo
+      name: podinfo
+      readOnly: false
+    - mountPath: /var/run/openvswitch/
+      name: shared-dir
+    - mountPath: /dev/hugepages
+      name: hugepage
+    resources:
+      requests:
+        cpu: "4000m"
+        hugepages-1Gi: 4Gi
+      limits:
+        cpu: "4000m"
+        hugepages-1Gi: 4Gi
+    command: ["sleep", "infinity"]
+  nodeSelector:
+    vswitch: ovs
+  volumes:
+  - name: podinfo
+    downwardAPI:
+      items:
+        - path: "labels"
+          fieldRef:
+            fieldPath: metadata.labels
+        - path: "annotations"
+          fieldRef:
+            fieldPath: metadata.annotations
+  - name: shared-dir
+    hostPath:
+      path: /usr/local/var/run/openvswitch/
+  - name: hugepage
+    emptyDir:
+      medium: HugePages
diff --git a/tools/k8s/reference-definitions/pod-defs/sriov/sriov-pod.yaml b/tools/k8s/reference-definitions/pod-defs/sriov/sriov-pod.yaml
new file mode 100644
index 00000000..2b5aff7a
--- /dev/null
+++ b/tools/k8s/reference-definitions/pod-defs/sriov/sriov-pod.yaml
@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sriov-pod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: sriov-net-a, sriov-net-b
+spec:
+  containers:
+  - name: sriov
+    image: dpdk-app-centos-numa1
+    imagePullPolicy: Never
+    securityContext:
+      privileged: true
+      capabilities:
+        add: ["CAP_SYS_ADMIN"]
+    volumeMounts:
+    - mountPath: /etc/podnetinfo
+      name: podnetinfo
+      readOnly: false
+    - mountPath: /dev/hugepages
+      name: hugepage
+    resources:
+      requests:
+        cpu: "6000m"
+        intel.com/intel_sriov_dpdk_a: '1'
+        intel.com/intel_sriov_dpdk_b: '1'
+      limits:
+        cpu: "6000m"
+        hugepages-1Gi: 2Gi
+        intel.com/intel_sriov_dpdk_a: '1'
+        intel.com/intel_sriov_dpdk_b: '1'
+    command: ["sleep", "infinity"]
+#  nodeSelector:
+#    vswitch: ovs
+  volumes:
+  - name: podnetinfo
+    downwardAPI:
+      items:
+        - path: "labels"
+          fieldRef:
+            fieldPath: metadata.labels
+        - path: "annotations"
+          fieldRef:
+            fieldPath: metadata.annotations
+  - name: hugepage
+    emptyDir:
+      medium: HugePages
diff --git a/tools/k8s/reference-definitions/pod-defs/vpp/userspace-dpdk-pod.yaml b/tools/k8s/reference-definitions/pod-defs/vpp/userspace-dpdk-pod.yaml
new file mode 100644
index 00000000..4b72df22
--- /dev/null
+++ b/tools/k8s/reference-definitions/pod-defs/vpp/userspace-dpdk-pod.yaml
@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: userspace-vpp-pod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: userspace-vpp-net, userspace-vpp-net
+spec:
+  containers:
+  - name: vpp-vhost
+    image: newdpdk-app-centos:latest
+    imagePullPolicy: Never
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/podnetinfo
+      name: podinfo
+      readOnly: false
+    - mountPath: /var/lib/cni/usrspcni/
+      name: shared-dir
+    - mountPath: /dev/hugepages
+      name: hugepage
+    resources:
+      requests:
+        cpu: "6000m"
+        memory: "4000Mi"
+        hugepages-1Gi: 4Gi
+      limits:
+        cpu: "6000m"
+        memory: "4000Mi"
+        hugepages-1Gi: 4Gi
+    command: ["sleep", "infinity"]
+  nodeSelector:
+    vswitch: ovs
+  volumes:
+  - name: podinfo
+    downwardAPI:
+      items:
+        - path: "labels"
+          fieldRef:
+            fieldPath: metadata.labels
+        - path: "annotations"
+          fieldRef:
+            fieldPath: metadata.annotations
+  - name: shared-dir
+    hostPath:
+      path: /var/lib/cni/usrspcni/data/
+  - name: hugepage
+    emptyDir:
+      medium: HugePages
diff --git a/tools/k8s/reference-definitions/pod-defs/vpp/userspace-dpdk-pod2.yaml b/tools/k8s/reference-definitions/pod-defs/vpp/userspace-dpdk-pod2.yaml
new file mode 100644
index 00000000..7e3d2e71
--- /dev/null
+++ b/tools/k8s/reference-definitions/pod-defs/vpp/userspace-dpdk-pod2.yaml
@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: userspace-vpp-pod2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: userspace-vpp-net, userspace-vpp-net
+spec:
+  containers:
+  - name: vpp-vhost
+    image: newdpdk-app-centos:latest
+    imagePullPolicy: Never
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/podnetinfo
+      name: podinfo
+      readOnly: false
+    - mountPath: /var/lib/cni/usrspcni/
+      name: shared-dir
+    - mountPath: /dev/hugepages
+      name: hugepage
+    resources:
+      requests:
+        cpu: "6000m"
+        hugepages-1Gi: 4Gi
+      limits:
+        cpu: "6000m"
+        hugepages-1Gi: 4Gi
+    command: ["sleep", "infinity"]
+  nodeSelector:
+    vswitch: ovs
+  volumes:
+  - name: podinfo
+    downwardAPI:
+      items:
+        - path: "labels"
+          fieldRef:
+            fieldPath: metadata.labels
+        - path: "annotations"
+          fieldRef:
+            fieldPath: metadata.annotations
+  - name: shared-dir
+    hostPath:
+      path: /var/lib/cni/usrspcni/data/
+  - name: hugepage
+    emptyDir:
+      medium: HugePages
diff --git a/tools/k8s/test-containers/dpdk-forwarding-pods/Dockerfile b/tools/k8s/test-containers/dpdk-forwarding-pods/Dockerfile
new file mode 100644
index 00000000..58f558fb
--- /dev/null
+++ b/tools/k8s/test-containers/dpdk-forwarding-pods/Dockerfile
@@ -0,0 +1,110 @@
+FROM centos:7
+
+#
+# Install required packages
+#
+RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo
+RUN yum groupinstall -y "Development Tools"
+RUN yum install -y wget numactl-devel git golang make; yum clean all
+# Debug Tools (if needed):
+#RUN yum install -y pciutils iproute; yum clean all
+
+#
+# Download and Build APP-NetUtil
+#
+WORKDIR /root/go/src/
+RUN go get github.com/openshift/app-netutil 2>&1 > /tmp/UserspaceDockerBuild.log || echo "Can ignore no GO files."
+WORKDIR /root/go/src/github.com/openshift/app-netutil
+RUN make c_sample
+RUN cp bin/libnetutil_api.so /lib64/libnetutil_api.so; cp bin/libnetutil_api.h /usr/include/libnetutil_api.h
+
+#
+# Download and Build DPDK
+#
+ENV DPDK_VER 20.05
+ENV DPDK_DIR /usr/src/dpdk-${DPDK_VER}
+WORKDIR /usr/src/
+RUN wget http://fast.dpdk.org/rel/dpdk-${DPDK_VER}.tar.xz
+RUN tar -xpvf dpdk-${DPDK_VER}.tar.xz
+
+ENV RTE_TARGET=x86_64-native-linuxapp-gcc
+ENV RTE_SDK=${DPDK_DIR}
+WORKDIR ${DPDK_DIR}
+# DPDK_VER 19.08
+RUN sed -i -e 's/EAL_IGB_UIO=y/EAL_IGB_UIO=n/' config/common_linux
+RUN sed -i -e 's/KNI_KMOD=y/KNI_KMOD=n/' config/common_linux
+RUN sed -i -e 's/LIBRTE_KNI=y/LIBRTE_KNI=n/' config/common_linux
+RUN sed -i -e 's/LIBRTE_PMD_KNI=y/LIBRTE_PMD_KNI=n/' config/common_linux
+# Additional Debug if Needed
+#RUN sed -i -e 's/CONFIG_RTE_LIBRTE_ETHDEV_DEBUG=n/CONFIG_RTE_LIBRTE_ETHDEV_DEBUG=y/' config/common_base
+
+# DPDK_VER 19.02
+#RUN sed -i -e 's/EAL_IGB_UIO=y/EAL_IGB_UIO=n/' config/common_linuxapp
+#RUN sed -i -e 's/KNI_KMOD=y/KNI_KMOD=n/' config/common_linuxapp
+#RUN sed -i -e 's/LIBRTE_KNI=y/LIBRTE_KNI=n/' config/common_linuxapp
+#RUN sed -i -e 's/LIBRTE_PMD_KNI=y/LIBRTE_PMD_KNI=n/' config/common_linuxapp
+
+# Add vhost patch
+COPY ./vhost_substitute.sh ./vhost_substitute.sh
+RUN ./vhost_substitute.sh
+
+RUN make install T=${RTE_TARGET} DESTDIR=${RTE_SDK}
+
+#
+# Build TestPmd
+#
+WORKDIR ${DPDK_DIR}/app/test-pmd
+COPY ./dpdk-args.c ./dpdk-args.c
+COPY ./dpdk-args.h ./dpdk-args.h
+COPY ./testpmd_eal_init.txt ./testpmd_eal_init.txt
+COPY ./testpmd_launch_args_parse.txt ./testpmd_launch_args_parse.txt
+COPY ./testpmd_substitute.sh ./testpmd_substitute.sh
+RUN ./testpmd_substitute.sh
+RUN make
+RUN cp testpmd /usr/bin/testpmd
+
+#
+# Build l2fwd
+#
+WORKDIR ${DPDK_DIR}/examples/l2fwd
+COPY ./dpdk-args.c ./dpdk-args.c
+COPY ./dpdk-args.h ./dpdk-args.h
+COPY ./l2fwd_eal_init.txt ./l2fwd_eal_init.txt
+COPY ./l2fwd_parse_args.txt ./l2fwd_parse_args.txt
+COPY ./l2fwd_substitute.sh ./l2fwd_substitute.sh
+RUN ./l2fwd_substitute.sh
+RUN make
+RUN cp build/l2fwd /usr/bin/l2fwd
+
+#
+# Build l3fwd
+#
+#WORKDIR ${DPDK_DIR}/examples/l3fwd
+#COPY ./dpdk-args.c ./dpdk-args.c
+#COPY ./dpdk-args.h ./dpdk-args.h
+#COPY ./l3fwd_eal_init.txt ./l3fwd_eal_init.txt
+#COPY ./l3fwd_parse_args.txt ./l3fwd_parse_args.txt
+#COPY ./l3fwd_substitute.sh ./l3fwd_substitute.sh
+#RUN ./l3fwd_substitute.sh
+#RUN make
+#RUN cp build/l3fwd /usr/bin/l3fwd
+
+# Copy default APP
+RUN cp /usr/bin/l2fwd /usr/bin/dpdk-app
+
+# -------- Import stage.
+# Docker 17.05 or higher
+# BEGIN
+FROM centos
+COPY --from=0 /usr/bin/dpdk-app /usr/bin/dpdk-app
+COPY --from=0 /usr/bin/l2fwd /usr/bin/l2fwd
+#COPY --from=0 /usr/bin/l3fwd /usr/bin/l3fwd
+COPY --from=0 /usr/bin/testpmd /usr/bin/testpmd
+COPY --from=0 /lib64/libnetutil_api.so /lib64/libnetutil_api.so
+COPY --from=0 /usr/lib64/libnuma.so.1 /usr/lib64/libnuma.so.1
+# END
+
+# COPY ./docker-entrypoint.sh /
+# RUN chmod +x /docker-entrypoint.sh
+# ENTRYPOINT ["sleep", "5s"]
+#CMD ["l2fwd"]
diff --git a/tools/k8s/test-containers/trafficgen-pods/pktgen/Dockerfile b/tools/k8s/test-containers/trafficgen-pods/pktgen/Dockerfile
new file mode 100644
index 00000000..8288cf63
--- /dev/null
+++ b/tools/k8s/test-containers/trafficgen-pods/pktgen/Dockerfile
@@ -0,0 +1,48 @@
+FROM centos:7 as builder
+
+#
+## Install required packages
+##
+RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo
+RUN yum groupinstall -y "Development Tools"
+RUN yum install -y wget numactl-devel git golang make; yum clean all
+## Debug Tools (if needed):
+RUN yum install -y pciutils iproute; yum clean all
+
+##
+## Download and Build APP-NetUtil
+##
+WORKDIR /root/go/src/
+RUN go get github.com/openshift/app-netutil 2>&1 > /tmp/UserspaceDockerBuild.log || echo "Can ignore no GO files."
+WORKDIR /root/go/src/github.com/openshift/app-netutil
+RUN make c_sample
+RUN cp bin/libnetutil_api.so /lib64/libnetutil_api.so; cp bin/libnetutil_api.h /usr/include/libnetutil_api.h
+
+## Download and Build DPDK
+##
+ENV DPDK_VER 21.02
+ENV DPDK_DIR /usr/src/dpdk-${DPDK_VER}
+WORKDIR /usr/src/
+RUN wget http://fast.dpdk.org/rel/dpdk-${DPDK_VER}.tar.xz
+RUN tar -xpvf dpdk-${DPDK_VER}.tar.xz
+ENV RTE_TARGET=x86_64-native-linuxapp-gcc
+ENV RTE_SDK=${DPDK_DIR}
+WORKDIR ${DPDK_DIR}
+
+COPY ./vhost_substitute.sh ./vhost_substitute.sh
+RUN ./vhost_substitute.sh
+
+RUN yum install -y epel-release && yum install -y dnf
+RUN dnf install -y meson ninja-build
+RUN pip3 install pyelftools
+
+RUN meson build && ninja -C build && ninja -C build install && ldconfig
+
+EXPOSE 22 8474
+
+WORKDIR /usr/src/
+ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig
+RUN git clone http://dpdk.org/git/apps/pktgen-dpdk
+RUN yum install -y libpcap libpcap-devel
+RUN cd pktgen-dpdk && make -j
+