diff options
| -rwxr-xr-x | functions.sh | 10 | ||||
| -rw-r--r-- | hw_config/equinix-metal/idf.yaml | 2 | ||||
| -rw-r--r-- | hw_config/intel/idf.yaml | 2 | ||||
| -rw-r--r-- | playbooks/roles/bmra-config/templates/all.j2 | 2 | ||||
| -rw-r--r-- | playbooks/roles/bmra-config/templates/inventory.j2 | 2 | ||||
| -rw-r--r-- | playbooks/roles/bmra-config/templates/kube-node.j2 | 22 | ||||
| -rw-r--r-- | sw_config/bmra/dpdk_patch.yml | 111 | ||||
| -rw-r--r-- | sw_config/bmra/patched_cmk.yml | 84 | ||||
| -rw-r--r-- | sw_config/bmra/patched_tas.yml | 179 |
9 files changed, 17 insertions, 397 deletions
diff --git a/functions.sh b/functions.sh index cb099d5..03e2284 100755 --- a/functions.sh +++ b/functions.sh @@ -194,7 +194,7 @@ provision_k8s() { ansible_cmd="/bin/bash -c '" if [[ "$DEPLOYMENT" == "k8s" ]]; then ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml - ansible_cmd+="pip install --upgrade pip==9.0.3; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;" + ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;" fi ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'" @@ -210,7 +210,7 @@ if ! command -v docker; then done fi if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then - git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.0.0 ${PROJECT_ROOT}/container-experience-kits/ + git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.1.0 ${PROJECT_ROOT}/container-experience-kits/ cp -r ${PROJECT_ROOT}/container-experience-kits/examples/${BMRA_PROFILE}/group_vars ${PROJECT_ROOT}/container-experience-kits/ # NOTE The following condition/workaround will be removed once the reported issue https://github.com/intel/container-experience-kits/issues/68 # is fixed upstream @@ -225,12 +225,6 @@ cp ${PROJECT_ROOT}/${INSTALLER}/inventory.ini \ ${PROJECT_ROOT}/container-experience-kits/ cp ${PROJECT_ROOT}/${INSTALLER}/{all.yml,kube-node.yml} \ ${PROJECT_ROOT}/container-experience-kits/group_vars/ -cp ${PROJECT_ROOT}/${INSTALLER}/dpdk_patch.yml \ - ${PROJECT_ROOT}/container-experience-kits/roles/install_dpdk/tasks/main.yml -cp ${PROJECT_ROOT}/${INSTALLER}/patched_tas.yml \ - ${PROJECT_ROOT}/container-experience-kits/roles/tas_install/tasks/tas.yml -cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk.yml \ - ${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/job.yml cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk_build.yml \ ${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/tasks/main.yml sudo docker run --rm \ diff --git a/hw_config/equinix-metal/idf.yaml b/hw_config/equinix-metal/idf.yaml index 434ada2..d085c7c 100644 --- a/hw_config/equinix-metal/idf.yaml +++ b/hw_config/equinix-metal/idf.yaml @@ -77,5 +77,5 @@ bmra: policy: "best-effort" tas: enable: true - demo_policy: true + demo_policy: false psp: true diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml index 13b8054..07c45cc 100644 --- a/hw_config/intel/idf.yaml +++ b/hw_config/intel/idf.yaml @@ -106,5 +106,5 @@ bmra: policy: "best-effort" tas: enable: true - demo_policy: true + demo_policy: false psp: true diff --git a/playbooks/roles/bmra-config/templates/all.j2 b/playbooks/roles/bmra-config/templates/all.j2 index 1017331..6dc074e 100644 --- a/playbooks/roles/bmra-config/templates/all.j2 +++ b/playbooks/roles/bmra-config/templates/all.j2 @@ -65,7 +65,7 @@ topology_manager_policy: "{{ bmra.features.topology_manager.policy }}" sriov_net_dp_enabled: {{ bmra.features.sriov_net_dp }} sriov_net_dp_namespace: kube-system # whether to build and store image locally or use one from public external registry -sriov_net_dp_build_image_locally: false +sriov_net_dp_build_image_locally: true # SR-IOV network device plugin configuration. # For more information on supported configuration refer to: https://github.com/intel/sriov-network-device-plugin#configurations sriovdp_config_data: | diff --git a/playbooks/roles/bmra-config/templates/inventory.j2 b/playbooks/roles/bmra-config/templates/inventory.j2 index 28581e0..6008179 100644 --- a/playbooks/roles/bmra-config/templates/inventory.j2 +++ b/playbooks/roles/bmra-config/templates/inventory.j2 @@ -2,7 +2,7 @@ {% for node in nodes %} {{ idf.kubespray.hostnames[node.name] }} ansible_host={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }} ip={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }} {% endfor %} -localhost +localhost ansible_python_interpreter=/usr/bin/python2 {% for role in ['kube-master', 'etcd', 'kube-node'] %} [{{ role }}] diff --git a/playbooks/roles/bmra-config/templates/kube-node.j2 b/playbooks/roles/bmra-config/templates/kube-node.j2 index 302fa8f..f32fbdb 100644 --- a/playbooks/roles/bmra-config/templates/kube-node.j2 +++ b/playbooks/roles/bmra-config/templates/kube-node.j2 @@ -16,11 +16,11 @@ --- # Kubernetes node configuration -# Enable SR-IOV networking related setup -sriov_enabled: {{ bmra.features.sriov.enable }} +# Enable IOMMU (required for SR-IOV networking) +iommu_enabled: {{ bmra.features.sriov.enable }} -# SR-IOV PF specific configuration list -sriov_nics: +# dataplane interface configuration list +dataplane_interfaces: {% for intf in bmra.network_roles.sriov %} - name: {{ intf.name }} bus_info: "{{ intf.bus_info }}" @@ -41,8 +41,8 @@ sriov_cni_enabled: {{ bmra.features.sriov_cni }} # Bond CNI bond_cni_enabled: false -# install DPDK -install_dpdk: true # DPDK installation is required for sriov_enabled:true; default to false +# Install DPDK (required for SR-IOV networking) +install_dpdk: true # Userspace networking userspace_cni_enabled: false @@ -62,10 +62,10 @@ update_nic_drivers: false update_nic_firmware: false # Additional list of NIC interfaces that the FW update will be executed on -# NOTE: FW update will be also executed on all NICs listed in "sriov_nics[*].name" -firmware_update_nics: - - intf1 - - intf2 +# NOTE: FW update will be also executed on all NICs listed in "dataplane_interfaces[*].name" +firmware_update_nics: [] +# - enp24s0f0 +# - enp24s0f1 # install Intel x700 & x800 series NICs DDP packages install_ddp_packages: false @@ -93,7 +93,7 @@ cpu_count: 144 # Enable/Disable Intel PState scaling driver intel_pstate_enabled: true # Config options for intel_pstate: disable, passive, force, no_hwp, hwp_only, support_acpi_ppc, per_cpu_perf_limits -intel_pstate: disable +intel_pstate: hwp_only # Enable/Disable Intel Turbo Boost PState attribute turbo_boost_enabled: false diff --git a/sw_config/bmra/dpdk_patch.yml b/sw_config/bmra/dpdk_patch.yml deleted file mode 100644 index ec77b39..0000000 --- a/sw_config/bmra/dpdk_patch.yml +++ /dev/null @@ -1,111 +0,0 @@ -## -## Copyright (c) 2020 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: install dependencies - include_role: - name: install_dependencies - -- name: download DPDK - unarchive: - src: "{{ dpdk_url }}" - dest: "/usr/src" - remote_src: yes - list_files: yes - mode: 0755 - register: dpdk_download - -- name: set local dpdk directory path - set_fact: - dpdk_dir: "{{ dpdk_download.dest }}/{{ dpdk_download.files[0] }}" - -- name: run make config - make: - chdir: "{{ dpdk_dir }}" - target: config - params: - T: "{{ dpdk_target }}" - -- name: update ansible_kernel fact - setup: - filter: 'ansible_kernel' - -- name: patch DPDK (kni) - lineinfile: - path: "{{ dpdk_dir }}/kernel/linux/kni/kni_net.c" - regexp: '^\s*kni_net_tx_timeout' - line: 'kni_net_tx_timeout(struct net_device *dev, unsigned int txqueue)' - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == '8' - - ansible_kernel is version('4.18.0-240','>=') - -- name: enable virtio-user support - lineinfile: - path: "{{ dpdk_dir }}/build/.config" - regexp: '^CONFIG_RTE_VIRTIO_USER' - line: 'CONFIG_RTE_VIRTIO_USER=y' - mode: 0600 - -- name: enable PCAP PMD support - lineinfile: - path: "{{ dpdk_dir }}/build/.config" - regexp: '^CONFIG_RTE_LIBRTE_PMD_PCAP' - line: 'CONFIG_RTE_LIBRTE_PMD_PCAP=y' - mode: 0600 - -- name: build DPDK - make: - target: install - chdir: "{{ dpdk_dir }}" - params: - T: "{{ dpdk_target }}" - DESTDIR: install - prefix: "/usr" - -- name: find dpdk tools - find: - path: "{{ dpdk_dir }}" - patterns: "dpdk-devbind.py" - recurse: yes - register: dpdk_tools_dir - -- name: set path to dpdk usertools directory - set_fact: - dpdk_tools: "{{ dpdk_tools_dir.files[0].path }}" - -- name: load userspace modules - modprobe: - name: "{{ item }}" - state: present - with_items: - - vfio-pci - - uio - -- name: install dpdk-devbind.py in /usr/local/bin - copy: - remote_src: yes - src: "{{ dpdk_tools }}" - dest: "/usr/local/bin/dpdk-devbind.py" - mode: 0700 - owner: root - group: root - become: yes - -- name: load intel module - command: "insmod {{ dpdk_dir }}/{{ dpdk_target }}/kmod/igb_uio.ko" - register: result - failed_when: "'No such file or directory' in result.stderr" - changed_when: "'already bound' not in result.stderr" diff --git a/sw_config/bmra/patched_cmk.yml b/sw_config/bmra/patched_cmk.yml deleted file mode 100644 index e97083a..0000000 --- a/sw_config/bmra/patched_cmk.yml +++ /dev/null @@ -1,84 +0,0 @@ -{{- $fullName := include "cmk.fullname" . -}} -{{- $exclusiveMode := .Values.exclusiveMode -}} -{{- $numExclusiveCores := .Values.numExclusiveCores -}} -{{- $sharedMode := .Values.sharedMode -}} -{{- $numSharedCores := .Values.numSharedCores -}} -{{- $pullPolicy := .Values.image.pullPolicy -}} -{{- $image := .Values.image.repository -}} -{{- $tag := .Values.image.tag -}} -{{- $noTaint := .Values.noTaint }} -{{ range splitList "," .Values.hosts.list }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "10" - labels: - app: {{ $fullName }}-init-discover-{{ . }} - name: {{ $fullName }}-init-discover-{{ . }} -spec: - template: - spec: - serviceAccountName: {{ $fullName }} - restartPolicy: Never - tolerations: - - key: cmk - operator: Exists - containers: - - name: install - image: {{ $image }}:{{ $tag }} - imagePullPolicy: {{ $pullPolicy }} - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py install" - volumeMounts: - - mountPath: /opt/bin - name: cmk-install-dir - - name: init - image: {{ $image }}:{{ $tag }} - imagePullPolicy: {{ $pullPolicy }} - env: - - name: CMK_PROC_FS - value: "/host/proc" - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py init --exclusive-mode={{ $exclusiveMode }} --num-exclusive-cores={{ $numExclusiveCores }} --shared-mode={{ $sharedMode }} --num-shared-cores={{ $numSharedCores }}" - volumeMounts: - - mountPath: /host/proc - name: host-proc - readOnly: true - - name: discover - image: {{ $image }}:{{ $tag }} - imagePullPolicy: {{ $pullPolicy }} - env: - - name: CMK_PROC_FS - value: /host/proc - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py discover {{ if $noTaint }}--no-taint{{ end }}" - volumeMounts: - - mountPath: /host/proc - name: host-proc - readOnly: true - volumes: - - hostPath: - path: /proc - type: Directory - name: host-proc - - hostPath: - path: /opt/bin - type: DirectoryOrCreate - name: cmk-install-dir -{{ end }} diff --git a/sw_config/bmra/patched_tas.yml b/sw_config/bmra/patched_tas.yml deleted file mode 100644 index 633f5d7..0000000 --- a/sw_config/bmra/patched_tas.yml +++ /dev/null @@ -1,179 +0,0 @@ -## -## Copyright (c) 2020 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name : install dependencies - include_role: - name: install_dependencies - -- name: update to git2 on RHEL 7 based distros - include_role: - name: git2_install - when: - - ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - - ansible_distribution_version < '8' - -- name: check if stable repo has already been added - command: helm repo list - register: helm_repo_list - failed_when: false - changed_when: false - -- name: add Helm stable repo - command: helm repo add stable https://charts.helm.sh/stable - when: '"https://charts.helm.sh/stable" not in helm_repo_list.stdout' - register: helm_add_result - changed_when: '"has been added to your repositories" in helm_add_result.stdout' - -- name: update Helm repo before installation of public charts - command: helm repo update - register: helm_update_result - changed_when: '"Successfully got an update" in helm_update_result.stdout' - - -- name: create Helm charts directory if needed - file: - path: /usr/src/charts - state: directory - mode: 0755 - -- name: generate cert and key - include_role: - name: create_signed_k8s_certs - vars: - secret_name: "{{ tas_extender_secret_name }}" - service_name: tas-telemetry-aware-scheduling - key_pair_name: tas - host_secrets_folder: "{{ tas_ssl_mount_path }}" - k8s_namespace: "{{ tas_namespace }}" - csr_cluster_name: "{{ cluster_name | default('cluster.local') }}" - when: tas_tls_enabled - -- name: clone TAS repository - git: - repo: "{{ tas_git_url }}" - version: "{{ tas_git_version }}" - dest: "{{ tas_dir }}" - force: yes - -- name: make build and make image - TAS - make: - target: "{{ item }}" - chdir: "{{ tas_dir }}" - loop: - - build - - image - -- name: tag TAS-controller and TAS-extender - # TAS Makefile always creates ":latest" version images - command: docker tag {{ item }}:latest {{ registry_local_address }}/{{ item }}:{{ tas_version }} - loop: - - tas-controller - - tas-extender - changed_when: false - -- name: push TAS-controller and TAS-extender image to local registry - command: docker push {{ registry_local_address }}/{{ item }}:{{ tas_version }} - loop: - - tas-controller - - tas-extender - changed_when: true - -- name: create descheduler directory if needed - file: - path: "{{ sigs_k8s_io_dir }}" - state: directory - mode: 0755 - -- name: clone Descheduler for Kubernetes - git: - repo: "{{ descheduler_git_url }}" - dest: "{{ descheduler_dir }}" - force: yes - version: "{{ descheduler_git_version }}" - -- name: install descheduler - make: - chdir: "{{ descheduler_dir }}" - -- name: copy Helm chart resource definition to controller node - copy: - src: "{{ role_path }}/charts/{{ item }}" - dest: "/usr/src/charts/" - mode: preserve - loop: - - telemetry-aware-scheduling - - tas-policy-crd.yml - -- name: populate tas Helm chart values template and push to controller node - template: - src: "tas-values.yml.j2" - dest: "/usr/src/charts/tas-values.yml" - force: yes - mode: preserve - -- name: create TASPolicy resource - command: kubectl apply -f tas-policy-crd.yml - args: - chdir: "/usr/src/charts" - changed_when: true - -- name: install TAS helm chart - command: helm upgrade --install --namespace {{ tas_namespace }} {{ tas_name }} -f tas-values.yml telemetry-aware-scheduling/ - args: - chdir: "/usr/src/charts" - retries: 5 - delay: 5 - register: result - until: result.rc == 0 - changed_when: true - -- name: Configure arguments from Kubernetes Scheduler file if they exist - dnsPolicy - lineinfile: - path: /etc/kubernetes/manifests/kube-scheduler.yaml - insertafter: "spec:" - line: " dnsPolicy: ClusterFirstWithHostNet" - regexp: " dnsPolicy: " - state: present - mode: 0600 - -- name: Configure arguments to our kube-scheduler manifest - configmap - lineinfile: - path: /etc/kubernetes/manifests/kube-scheduler.yaml - insertafter: " - kube-scheduler" - line: "{{ item.arg }}={{ item.value }}" - regexp: "{{ item.arg }}" - state: present - mode: 0600 - with_items: - - { arg: " - --policy-configmap", value: "{{ tas_name }}-telemetry-aware-scheduling-scheduler-extender-policy" } - - { arg: " - --policy-configmap-namespace", value: "{{ tas_namespace }}" } - -# TAS Demo Policy -- name: template TAS demo policy - template: - src: "tas-demo-policy.yml.j2" - dest: "/usr/src/charts/tas-demo-policy.yml" - force: yes - mode: preserve - when: - - tas_enable_demo_policy - -- name: create TAS demo policy resource - command: kubectl apply -f tas-demo-policy.yml - args: - chdir: "/usr/src/charts" - when: - - tas_enable_demo_policy |