diff options
| -rwxr-xr-x | build/bin/entrypoint | 111 | ||||
| -rw-r--r-- | deploy/ovn-daemonset.yaml | 239 | ||||
| -rw-r--r-- | utilities/docker/debian/Dockerfile | 40 | ||||
| -rwxr-xr-x | utilities/docker/debian/ovn4nfv-k8s.sh | 158 |
4 files changed, 527 insertions, 21 deletions
diff --git a/build/bin/entrypoint b/build/bin/entrypoint index 77084a3..c9646a0 100755 --- a/build/bin/entrypoint +++ b/build/bin/entrypoint @@ -1,32 +1,101 @@ -#!/bin/sh -e +#!/bin/bash +set -e +CNI_VERSION=${CNI_VERSION:-"v0.8.5"} +IMAGE_ARC=${IMAGE_ARC:-"amd64"} + +create_kubeconfig() { + # Make a ovn4nfv.d directory (for our kubeconfig) + # Inspired from t.ly/Xgbbe + mkdir -p $CNI_CONF_DIR/ovn4nfv-k8s.d + OVN4NFV_KUBECONFIG=$CNI_CONF_DIR/ovn4nfv-k8s.d/ovn4nfv-k8s.kubeconfig + SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount + KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt} + SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token) + SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false} + + # Check if we're running as a k8s pod. + if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then + # We're running as a k8d pod - expect some variables. + if [ -z ${KUBERNETES_SERVICE_HOST} ]; then + error "KUBERNETES_SERVICE_HOST not set"; exit 1; + fi + if [ -z ${KUBERNETES_SERVICE_PORT} ]; then + error "KUBERNETES_SERVICE_PORT not set"; exit 1; + fi + + if [ "$SKIP_TLS_VERIFY" == "true" ]; then + TLS_CFG="insecure-skip-tls-verify: true" + elif [ -f "$KUBE_CA_FILE" ]; then + TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')" + fi + + # Write a kubeconfig file for the CNI plugin. Do this + # to skip TLS verification for now. We should eventually support + # writing more complete kubeconfig files. This is only used + # if the provided CNI network config references it. + touch $OVN4NFV_KUBECONFIG + chmod ${KUBECONFIG_MODE:-600} $OVN4NFV_KUBECONFIG + cat > $OVN4NFV_KUBECONFIG <<EOF +# Kubeconfig file for OVN4NFV-K8S CNI plugin. +apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://[${KUBERNETES_SERVICE_HOST}]:${KUBERNETES_SERVICE_PORT} + $TLS_CFG +users: +- name: ovn4nfv + user: + token: "${SERVICEACCOUNT_TOKEN}" +contexts: +- name: ovn4nfv-context + context: + cluster: local + user: ovn4nfv +current-context: ovn4nfv-context +EOF + else + warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)" + fi +} + +install_cni_plugins() { + curl --insecure --compressed -O -L https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION/cni-plugins-linux-$IMAGE_ARC-$CNI_VERSION.tgz + tar -zxvf cni-plugins-linux-$IMAGE_ARC-$CNI_VERSION.tgz -C $CNI_BIN_DIR + rm -rf cni-plugins-linux-$IMAGE_ARC-$CNI_VERSION.tgz +} cmd=${1:-""} case ${cmd} in - "cni") - CNI_BIN_DIR="/host/opt/cni/bin" - OVN4NFV_CONF_DIR="/host/etc/openvswitch" - OVN4NFV_BIN_FILE="/usr/local/bin/ovn4nfvk8s-cni" - OVN4NFV_CONF_FILE="/tmp/ovn4nfv-conf/ovn4nfv_k8s.conf" - cp -f $OVN4NFV_BIN_FILE $CNI_BIN_DIR - cp -f $OVN4NFV_CONF_FILE $OVN4NFV_CONF_DIR - # Sleep forever. - sleep infinity - ;; + "cni") + CNI_BIN_DIR="/host/opt/cni/bin" + OVN4NFV_CONF_DIR="/host/etc/openvswitch" + OVN4NFV_BIN_FILE="/usr/local/bin/ovn4nfvk8s-cni" + OVN4NFV_CONF_FILE="/tmp/ovn4nfv-conf/ovn4nfv_k8s.conf" + OVN4NFV_NET_CONF_FILE="/tmp/ovn4nfv-cni/00-network.conf" + CNI_CONF_DIR="/host/etc/cni/net.d" + + cp -f $OVN4NFV_BIN_FILE $CNI_BIN_DIR + cp -f $OVN4NFV_CONF_FILE $OVN4NFV_CONF_DIR + cp -f $OVN4NFV_NET_CONF_FILE $CNI_CONF_DIR + create_kubeconfig + install_cni_plugins + # Sleep forever. + sleep infinity + ;; "operator") - shift - exec ${OPERATOR} $@ - ;; + shift + exec ${OPERATOR} $@ + ;; "agent") - shift - exec ${AGENT} $@ - ;; - - + shift + exec ${AGENT} $@ + ;; *) - echo "invalid command ${cmd}" - + echo "invalid command ${cmd}" esac diff --git a/deploy/ovn-daemonset.yaml b/deploy/ovn-daemonset.yaml new file mode 100644 index 0000000..bb64d84 --- /dev/null +++ b/deploy/ovn-daemonset.yaml @@ -0,0 +1,239 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-nb-tcp + namespace: kube-system +spec: + ports: + - name: ovn-nb-tcp + protocol: TCP + port: 6641 + targetPort: 6641 + type: ClusterIP + selector: + app: ovn-control-plane + sessionAffinity: None + +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-sb-tcp + namespace: kube-system +spec: + ports: + - name: ovn-sb-tcp + protocol: TCP + port: 6642 + targetPort: 6642 + type: ClusterIP + selector: + app: ovn-control-plane + sessionAffinity: None + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-control-plane + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN control plane deployment using tcp: ovn-northd-tcp, ovn-nb-tcp and ovn-sb-tcp. +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 100% + type: RollingUpdate + selector: + matchLabels: + app: ovn-control-plane + template: + metadata: + labels: + app: ovn-control-plane + spec: + tolerations: + - operator: Exists + effect: NoSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-control-plane + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + hostNetwork: true + containers: + - name: ovn-control-plane + image: integratedcloudnative/ovn-images:master + imagePullPolicy: "IfNotPresent" + command: ["ovn4nfv-k8s", "start_ovn_control_plane"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 500m + memory: 300Mi + volumeMounts: + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + readinessProbe: + exec: + command: ["ovn4nfv-k8s", "check_ovn_control_plane"] + periodSeconds: 3 + livenessProbe: + exec: + command: ["ovn4nfv-k8s", "check_ovn_control_plane"] + initialDelaySeconds: 30 + periodSeconds: 7 + failureThreshold: 5 + nodeSelector: + beta.kubernetes.io/os: "linux" + ovn4nfv-k8s-plugin: ovn-control-plane + volumes: + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovn-controller + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN controller: Start ovsdb-server & ovs-vswitchd components, and ovn controller +spec: + selector: + matchLabels: + app: ovn-controller + updateStrategy: + type: OnDelete + template: + metadata: + labels: + app: ovn-controller + spec: + tolerations: + - operator: Exists + effect: NoSchedule + priorityClassName: system-cluster-critical + hostNetwork: true + hostPID: true + containers: + - name: ovn-controller + image: integratedcloudnative/ovn-images:master + imagePullPolicy: IfNotPresent + command: ["ovn4nfv-k8s", "start_ovn_controller"] + securityContext: + runAsUser: 0 + privileged: true + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + readinessProbe: + exec: + command: ["ovn4nfv-k8s", "check_ovn_controller"] + periodSeconds: 5 + livenessProbe: + exec: + command: ["ovn4nfv-k8s", "check_ovn_controller"] + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 5 + resources: + requests: + cpu: 200m + memory: 300Mi + limits: + cpu: 1000m + memory: 800Mi + nodeSelector: + beta.kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn diff --git a/utilities/docker/debian/Dockerfile b/utilities/docker/debian/Dockerfile new file mode 100644 index 0000000..674ee7e --- /dev/null +++ b/utilities/docker/debian/Dockerfile @@ -0,0 +1,40 @@ +FROM ubuntu:18.04 as base + +USER root + +RUN apt-get update && apt-get install -y iproute2 curl software-properties-common setpriv dpkg-dev netcat + +RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb +RUN bash -xc "\ +pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch-dev_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-common_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-dkms_2.12.0-1_all.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-source_2.12.0-1_all.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-dbg_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-ipsec_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-pki_2.12.0-1_all.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-switch_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-testcontroller_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-test_2.12.0-1_all.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-vtep_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-central_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-common_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-controller-vtep_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-docker_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-host_2.12.0-1_amd64.deb; \ +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/python-openvswitch_2.12.0-1_all.deb; \ +dpkg-scanpackages . | gzip -c9 > Packages.gz; \ +popd; \ +" +RUN ls -lt /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb +RUN echo "deb [trusted=yes] file:///opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb ./" | tee -a /etc/apt/sources.list > /dev/null +RUN apt-get update +RUN apt-get install -y openvswitch-switch=2.12.0-1 openvswitch-common=2.12.0-1 ovn-central=2.12.0-1 ovn-common=2.12.0-1 ovn-host=2.12.0-1 +RUN mkdir -p /var/run/openvswitch && \ + mkdir -p /var/run/ovn + +COPY ovn4nfv-k8s.sh /usr/local/bin/ovn4nfv-k8s + +ENTRYPOINT ["ovn4nfv-k8s"] diff --git a/utilities/docker/debian/ovn4nfv-k8s.sh b/utilities/docker/debian/ovn4nfv-k8s.sh new file mode 100755 index 0000000..ee60e2b --- /dev/null +++ b/utilities/docker/debian/ovn4nfv-k8s.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +OVS_RUNDIR=/var/run/openvswitch +OVS_LOGDIR=/var/log/openvswitch + +DB_NB_ADDR=${DB_NB_ADDR:-::} +DB_NB_PORT=${DB_NB_PORT:-6641} +DB_SB_ADDR=${DB_SB_ADDR:-::} +DB_SB_PORT=${DB_SB_PORT:-6642} +cmd=${1:-""} + +if [[ -f /usr/bin/ovn-appctl ]] ; then + # ovn-appctl is present. Use new ovn run dir path. + OVN_RUNDIR=/var/run/ovn + OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl + OVN_LOGDIR=/var/log/ovn + OVN_ETCDIR=/etc/ovn +else + # ovn-appctl is not present. Use openvswitch run dir path. + OVN_RUNDIR=/var/run/openvswitch + OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl + OVN_LOGDIR=/var/log/openvswitch + OVN_ETCDIR=/etc/openvswitch +fi + +check_ovn_control_plane() { + /usr/share/ovn/scripts/ovn-ctl status_northd + /usr/share/ovn/scripts/ovn-ctl status_ovnnb + /usr/share/ovn/scripts/ovn-ctl status_ovnsb +} + +check_ovn_controller() { + /usr/share/ovn/scripts/ovn-ctl status_controller +} + +# wait for ovn-sb ready +wait_ovn_sb() { + if [[ -z "${OVN_SB_TCP_SERVICE_HOST}" ]]; then + echo "env OVN_SB_SERVICE_HOST not exists" + exit 1 + fi + if [[ -z "${OVN_SB_TCP_SERVICE_PORT}" ]]; then + echo "env OVN_SB_SERVICE_PORT not exists" + exit 1 + fi + while ! nc -z "${OVN_SB_TCP_SERVICE_HOST}" "${OVN_SB_TCP_SERVICE_PORT}" </dev/null; + do + echo "sleep 10 seconds, waiting for ovn-sb ${OVN_SB_TCP_SERVICE_HOST}:${OVN_SB_TCP_SERVICE_PORT} ready " + sleep 10; + done +} + +start_ovs_vswitch() { + wait_ovn_sb + function quit { + /usr/share/openvswitch/scripts/ovs-ctl stop + /usr/share/openvswitch/scripts/ovn-ctl stop_controller + exit 0 + } + trap quit EXIT + /usr/share/openvswitch/scripts/ovs-ctl restart --no-ovs-vswitchd --system-id=random + # Restrict the number of pthreads ovs-vswitchd creates to reduce the + # amount of RSS it uses on hosts with many cores + # https://bugzilla.redhat.com/show_bug.cgi?id=1571379 + # https://bugzilla.redhat.com/show_bug.cgi?id=1572797 + if [[ `nproc` -gt 12 ]]; then + ovs-vsctl --no-wait set Open_vSwitch . other_config:n-revalidator-threads=4 + ovs-vsctl --no-wait set Open_vSwitch . other_config:n-handler-threads=10 + fi + + # Start ovsdb + /usr/share/openvswitch/scripts/ovs-ctl restart --no-ovsdb-server --system-id=random + /usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol + +} + +#cleanup_ovs_server() { +#} + +#cleanup_ovs_controller() { +#} + +function get_default_inteface_ipaddress { + local _ip=$1 + local _default_interface=$(awk '$2 == 00000000 { print $1 }' /proc/net/route) + local _ipv4address=$(ip addr show dev $_default_interface | awk '$1 == "inet" { sub("/.*", "", $2); print $2 }') + eval $_ip="'$_ipv4address'" +} + +start_ovn_control_plane() { + function quit { + /usr/share/openvswitch/scripts/ovn-ctl stop_northd + exit 0 + } + trap quit EXIT + /usr/share/openvswitch/scripts/ovn-ctl restart_northd + ovn-nbctl set-connection ptcp:"${DB_NB_PORT}":["${DB_NB_ADDR}"] + ovn-nbctl set Connection . inactivity_probe=0 + ovn-sbctl set-connection ptcp:"${DB_SB_PORT}":["${DB_SB_ADDR}"] + ovn-sbctl set Connection . inactivity_probe=0 + tail -f /var/log/openvswitch/ovn-northd.log +} + +start_ovn_controller() { + function quit { + /usr/share/openvswitch/scripts/ovn-ctl stop_controller + exit 0 + } + trap quit EXIT + wait_ovn_sb + get_default_inteface_ipaddress node_ipv4_address + /usr/share/openvswitch/scripts/ovn-ctl restart_controller + # Set remote ovn-sb for ovn-controller to connect to + ovs-vsctl set open . external-ids:ovn-remote=tcp:"${OVN_SB_TCP_SERVICE_HOST}":"${OVN_SB_TCP_SERVICE_PORT}" + ovs-vsctl set open . external-ids:ovn-remote-probe-interval=10000 + ovs-vsctl set open . external-ids:ovn-openflow-probe-interval=180 + ovs-vsctl set open . external-ids:ovn-encap-type=geneve + ovs-vsctl set open . external-ids:ovn-encap-ip=$node_ipv4_address + tail -f /var/log/openvswitch/ovn-controller.log +} + +set_nbclt() { + wait_ovn_sb + ovn-nbctl --db=tcp:["${OVN_NB_TCP_SERVICE_HOST}"]:"${OVN_NB_TCP_SERVICE_PORT}" --pidfile --detach --overwrite-pidfile +} + +check_ovs_vswitch() { + /usr/share/openvswitch/scripts/ovs-ctl status +} + +case ${cmd} in + "start_ovn_control_plane") + start_ovn_control_plane + ;; + "check_ovn_control_plane") + check_ovn_control_plane + ;; + "start_ovn_controller") + start_ovs_vswitch + set_nbclt + start_ovn_controller + ;; + "check_ovs_vswitch") + check_ovs_vswitch + ;; + "check_ovn_controller") + check_ovs_vswitch + check_ovn_controller + ;; + "cleanup_ovs_controller") + cleanup_ovs_controller + ;; + *) + echo "invalid command ${cmd}" + echo "valid commands: start-ovn-control-plane check_ovn_control_plane start-ovs-vswitch" + exit 0 +esac + +exit 0 |