diff options
| -rwxr-xr-x | functions.sh | 29 | ||||
| -rw-r--r-- | hw_config/equinix-metal/idf.yaml | 5 | ||||
| -rw-r--r-- | hw_config/ericsson-pod1/idf.yaml | 5 | ||||
| -rw-r--r-- | hw_config/intel/idf.yaml | 5 | ||||
| -rw-r--r-- | hw_config/libvirt-vms/idf.yaml | 9 | ||||
| -rw-r--r-- | playbooks/roles/bmra-config/templates/all.j2 | 42 | ||||
| -rw-r--r-- | playbooks/roles/bmra-config/templates/kube-node.j2 | 49 | ||||
| -rw-r--r-- | sw_config/bmra/Dockerfile | 2 | ||||
| -rw-r--r-- | sw_config/bmra/patched_cmk_build.yml | 65 | ||||
| -rw-r--r-- | sw_config/bmra/patched_k8s.yml | 30 | ||||
| -rw-r--r-- | sw_config/bmra/patched_kubespray_requirements.txt | 6 | ||||
| -rw-r--r-- | sw_config/bmra/patched_packages.yml | 7 | ||||
| -rw-r--r-- | sw_config/bmra/patched_preflight.yml | 544 | ||||
| -rw-r--r-- | sw_config/bmra/patched_rhel_packages.yml | 85 | ||||
| -rw-r--r-- | sw_config/bmra/patched_sriov_cni_install.yml | 44 |
15 files changed, 818 insertions, 109 deletions
diff --git a/functions.sh b/functions.sh index 69a3de7..89c1ebb 100755 --- a/functions.sh +++ b/functions.sh @@ -166,9 +166,9 @@ provision_hosts_baremetal() { if [ ! -d "${PROJECT_ROOT}/engine" ]; then ssh-keygen -t rsa -N "" -f "${PROJECT_ROOT}"/.ssh/id_rsa git clone https://gerrit.nordix.org/infra/engine.git - cp "${PROJECT_ROOT}"/"${VENDOR}"/{pdf.yaml,idf.yaml} \ - "${PROJECT_ROOT}"/engine/engine fi +cp "${PROJECT_ROOT}"/"${VENDOR}"/{pdf.yaml,idf.yaml} \ +"${PROJECT_ROOT}"/engine/engine cd "${PROJECT_ROOT}"/engine/engine || return ./deploy.sh -s ironic -d "${DISTRO}" \ -p file:///"${PROJECT_ROOT}"/engine/engine/pdf.yaml \ @@ -181,9 +181,9 @@ provision_hosts_vms() { # Install and run cloud-infra if [ ! -d "$CURRENTPATH/engine" ]; then git clone https://gerrit.nordix.org/infra/engine.git "${CURRENTPATH}"/engine - cp "$CURRENTPATH"/hw_config/"$VENDOR"/{pdf.yaml,idf.yaml} \ - "${CURRENTPATH}"/engine/engine fi +cp "$CURRENTPATH"/hw_config/"$VENDOR"/{pdf.yaml,idf.yaml} \ +"${CURRENTPATH}"/engine/engine cd "$CURRENTPATH"/engine/engine || return ./deploy.sh -s ironic \ -p file:///"${CURRENTPATH}"/engine/engine/pdf.yaml \ @@ -211,7 +211,7 @@ provision_k8s_baremetal() { ansible_cmd="/bin/bash -c '" if [[ "$DEPLOYMENT" == "k8s" ]]; then ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml - ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.17; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;" + ansible_cmd+="yum -y remove python-netaddr; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;" fi ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'" @@ -227,7 +227,7 @@ if ! command -v docker; then done fi if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then - git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 "${PROJECT_ROOT}"/container-experience-kits/ + git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${PROJECT_ROOT}"/container-experience-kits/ cp -r "${PROJECT_ROOT}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${PROJECT_ROOT}"/container-experience-kits/ fi if [ -f "${PROJECT_ROOT}/docker_config" ]; then @@ -252,13 +252,17 @@ cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_packages.yml \ "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_kubespray_requirements.txt \ "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt +cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_preflight.yml \ + "${PROJECT_ROOT}"/container-experience-kits/playbooks/preflight.yml +cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_sriov_cni_install.yml \ + "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml sudo docker run --rm \ -e ANSIBLE_CONFIG=/bmra/ansible.cfg \ -e PROFILE="${BMRA_PROFILE}" \ -v "${PROJECT_ROOT}"/container-experience-kits:/bmra \ --v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.03-install:centos \ -"${ansible_cmd}" +-v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \ +${ansible_cmd} EOF } @@ -266,7 +270,7 @@ provision_k8s_vms() { # shellcheck disable=SC2087 # Install BMRA if [ ! -d "${CURRENTPATH}/container-experience-kits" ]; then - git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 "${CURRENTPATH}"/container-experience-kits/ + git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${CURRENTPATH}"/container-experience-kits/ cp -r "${CURRENTPATH}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${CURRENTPATH}"/container-experience-kits/ fi cp "${CURRENTPATH}"/sw_config/bmra/{inventory.ini,ansible.cfg} \ @@ -283,14 +287,19 @@ cp "${CURRENTPATH}"/sw_config/bmra/patched_packages.yml \ "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_kubespray_requirements.txt \ "${CURRENTPATH}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt +cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_preflight.yml \ + "${CURRENTPATH}"/container-experience-kits/playbooks/preflight.yml +cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_sriov_cni_install.yml \ + "${CURRENTPATH}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml +# Ansible upgrade below can be removed once image is updated sudo docker run --rm \ -e ANSIBLE_CONFIG=/bmra/ansible.cfg \ -e PROFILE="${BMRA_PROFILE}" \ -v "${CURRENTPATH}"/container-experience-kits:/bmra \ --v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.03-install:centos \ +-v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \ ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/"${BMRA_PROFILE}".yml } diff --git a/hw_config/equinix-metal/idf.yaml b/hw_config/equinix-metal/idf.yaml index 0cdcc41..a1f9ef8 100644 --- a/hw_config/equinix-metal/idf.yaml +++ b/hw_config/equinix-metal/idf.yaml @@ -52,9 +52,9 @@ bmra: # qat: # - name: crypto01 # pci: "0000:ab:00.0" -# mod_type: qat_c62x # pci_type: c6xx # vfs: 4 + runtime: docker # 'docker' and 'containerd' supported features: sriov: enable: true @@ -65,8 +65,7 @@ bmra: hugepages: enable: true default: 2M - hugepages_1G: 0 - hugepages_2M: 10240 + amount: 10240 isolcpus: enable: true autogenerate: true diff --git a/hw_config/ericsson-pod1/idf.yaml b/hw_config/ericsson-pod1/idf.yaml index 411d36d..3e03d4c 100644 --- a/hw_config/ericsson-pod1/idf.yaml +++ b/hw_config/ericsson-pod1/idf.yaml @@ -81,9 +81,9 @@ bmra: # qat: # - name: crypto01 # pci: "0000:ab:00.0" -# mod_type: qat_c62x # pci_type: c6xx # vfs: 4 + runtime: docker # 'docker' and 'containerd' supported features: sriov: enable: false @@ -94,8 +94,7 @@ bmra: hugepages: enable: false default: 2M - hugepages_1G: 0 - hugepages_2M: 10240 + amount: 10240 isolcpus: enable: true autogenerate: true diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml index 72f8c95..dc95638 100644 --- a/hw_config/intel/idf.yaml +++ b/hw_config/intel/idf.yaml @@ -81,9 +81,9 @@ bmra: # qat: # - name: crypto01 # pci: "0000:ab:00.0" -# mod_type: qat_c62x # pci_type: c6xx # vfs: 4 + runtime: docker # 'docker' and 'containerd' supported features: sriov: enable: true @@ -94,8 +94,7 @@ bmra: hugepages: enable: true default: 2M - hugepages_1G: 0 - hugepages_2M: 10240 + amount: 10240 isolcpus: enable: true autogenerate: true diff --git a/hw_config/libvirt-vms/idf.yaml b/hw_config/libvirt-vms/idf.yaml index 1616f9f..ab4fe0a 100644 --- a/hw_config/libvirt-vms/idf.yaml +++ b/hw_config/libvirt-vms/idf.yaml @@ -77,9 +77,9 @@ bmra: # qat: # - name: crypto01 # pci: "0000:ab:00.0" -# mod_type: qat_c62x # pci_type: c6xx # vfs: 4 + runtime: docker # 'docker' and 'containerd' supported features: sriov: enable: false @@ -90,12 +90,11 @@ bmra: hugepages: enable: true default: 2M - hugepages_1G: 0 - hugepages_2M: 5120 + amount: 5120 isolcpus: - enable: false + enable: true autogenerate: false - cpus: "4-19,24-39,44-59,64-79" # Not used when autogenerate is true + cpus: "2-5" # Not used when autogenerate is true nfd: true cmk: enable: true diff --git a/playbooks/roles/bmra-config/templates/all.j2 b/playbooks/roles/bmra-config/templates/all.j2 index 1eca556..440b373 100644 --- a/playbooks/roles/bmra-config/templates/all.j2 +++ b/playbooks/roles/bmra-config/templates/all.j2 @@ -9,9 +9,12 @@ SPDX-License-Identifier: Apache-2.0 # Kubernetes version kubernetes: true -#kube_version: v1.20.4 -kube_version: v1.19.8 -#kube_version: v1.18.16 +kube_version: v1.21.1 +#kube_version: v1.20.6 +#kube_version: v1.19.8 + +# Kubernetes container runtime: docker, containerd +container_runtime: {{ bmra.runtime }} # Run system-wide package update (apt dist-upgrade, yum update, ...) # Note: enabling this may lead to unexpected results @@ -47,6 +50,10 @@ topology_manager_enabled: {{ bmra.features.topology_manager.enable }} # There are four supported policies: none, best-effort, restricted, single-numa-node. topology_manager_policy: "{{ bmra.features.topology_manager.policy }}" +# OpenShift SRIOV Network Operator +sriov_network_operator_enabled: false +sriov_network_operator_namespace: "sriov-network-operator" + # Intel SRIOV Network Device Plugin sriov_net_dp_enabled: {{ bmra.features.sriov_net_dp }} sriov_net_dp_namespace: kube-system @@ -91,19 +98,26 @@ sriovdp_config_data: | ] } +# Intel Device Plugin Operator +intel_dp_namespace: kube-system # namespace will be applied for SGX DP and GPU DP + # Intel QAT Device Plugin for Kubernetes qat_dp_enabled: {{ bmra.features.qat.enable }} qat_dp_namespace: kube-system qat_dp_build_image_locally: true +# This feature will enable OpenSSL*Engine +openssl_engine_enabled: false # To activate OpenSSL*Engine, install_openssl & update_qat_drivers must set to ‘true’ in host_vars + # Intel GPU Device Plugin for Kubernetes gpu_dp_enabled: false -gpu_dp_namespace: kube-system +gpu_dp_kernel_version: "5.4.48+" gpu_dp_build_image_locally: true # Intel SGX Device Plugin for Kubernetes sgx_dp_enabled: false sgx_dp_build_image_locally: true +sgx_aesmd_namespace: kube-system # ProvisionLimit is a number of containers that can share # the same SGX provision device. sgx_dp_provision_limit: 20 @@ -111,6 +125,16 @@ sgx_dp_provision_limit: 20 # same SGX enclave device. sgx_dp_enclave_limit: 20 +# KMRA (Key Management Reference Application) +kmra_enabled: false +# The PCCS uses this API key to request collaterals from Intel's Provisioning Certificate Service. +# User needs to subscribe first to obtain an API key. +# For how to subscribe to Intel Provisioning Certificate Service and receive an API key, +# goto https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'. +kmra_pccs_api_key: "ffffffffffffffffffffffffffffffff" +# deploy KMRA demo workload (NGINX server) +kmra_deploy_demo_workload: true + # Intel Telemetry Aware Scheduling tas_enabled: {{ bmra.features.tas.enable }} tas_namespace: monitoring @@ -153,9 +177,17 @@ helm_enabled: true # local Docker Hub mirror, if it exists #docker_registry_mirrors: # - http://mirror_ip:mirror_port +#containerd_registries: +# "docker.io": +# - "https://registry-1.docker.io" +# - "https://mirror_ip:mirror_port" # Docker registry running on the cluster allows us to store images not avaialble on Docker Hub, e.g. CMK -registry_local_address: "localhost:30500" +# The range of valid ports is 30000-32767 +registry_nodeport: 30500 +{% raw %} +registry_local_address: "localhost:{{ registry_nodeport }}" +{% endraw %} # Enable Pod Security Policy. This option enables PSP admission controller and creates minimal set of rules. psp_enabled: {{ bmra.features.psp }} diff --git a/playbooks/roles/bmra-config/templates/kube-node.j2 b/playbooks/roles/bmra-config/templates/kube-node.j2 index 2f66c5a..51c4112 100644 --- a/playbooks/roles/bmra-config/templates/kube-node.j2 +++ b/playbooks/roles/bmra-config/templates/kube-node.j2 @@ -40,22 +40,22 @@ bond_cni_enabled: {{ bmra.features.bond_cni }} install_dpdk: true # DPDK version -dpdk_version: "19.11.6" +dpdk_version: "21.05" # Custom DPDK patches local path -# dpdk_local_patches_dir: "/tmp/patches/dpdk-19.11.6" +# dpdk_local_patches_dir: "/tmp/patches/dpdk-20.11" # Userspace networking userspace_cni_enabled: false -ovs_dpdk_enabled: false # Should be enabled with Userspace CNI, when VPP is set to "false"; 1G hugepages required -ovs_version: "v2.13.0" +ovs_dpdk_enabled: false # Should be enabled with Userspace CNI, when VPP is set to "false"; 1G default_hugepage_size required +ovs_version: "v2.15.0" # CPU mask for OVS-DPDK PMD threads ovs_dpdk_lcore_mask: 0x1 # Huge memory pages allocated by OVS-DPDK per NUMA node in megabytes # example 1: "256,512" will allocate 256MB from node 0 abd 512MB from node 1 -# example 2: "1024" will allocate 1GB fron node 0 on a single socket board, e.g. in a VM +# example 2: "1024" will allocate 1GB from node 0 on a single socket board, e.g. in a VM ovs_dpdk_socket_mem: "256,0" -vpp_enabled: false # Should be enabled with Userspace CNI, when ovs_dpdk is set to "false"; 2M hugepages required +vpp_enabled: false # Should be enabled with Userspace CNI, when ovs_dpdk is set to "false"; 2M default_hugepage_size required # Set to 'true' to update i40e, ice and iavf kernel modules update_nic_drivers: false @@ -82,30 +82,15 @@ qat_devices: {% for dev in bmra.device_roles.qat %} - qat_dev: {{ dev.name }} qat_id: "{{ dev.pci }}" - qat_module_type: {{ dev.mod_type }} qat_pci_type: {{ dev.pci_type }} qat_sriov_numvfs: {{ dev.vfs }} {% endfor %} {% else %} qat_devices: [] {% endif %} -# - qat_dev: crypto01 # Device name as separate QAT Symmetric Crypto devices on which qat_sriov_numvfs will be distributed -# qat_id: "0000:ab:00.0" # Device QAT id one using DPDK compatible driver for VF devices to be used by vfio-pci kernel driver, replace as required -# qat_module_type: qat_c62x # QAT Crypto Poll Mode Kernel Module supported are qat_dh895xcc,qat_c62x,qat_c3xxx,qat_200xx,qat_c4xxx,qat_d15xx -# qat_pci_type: c6xx # QAT Crypto Poll Mode Pci Driver id supported are dh895xcc,c6xx,c3xxx,d15xx,200xx & c4xxx -# qat_sriov_numvfs: 12 # set how many VFs per qat_id to be created such as c6xxvf support 32 so per device will be 10+10+12=32, replace as required - # Note: If don't want to assign VFs to id leave it as 0 such as qat_sriov_numvfs: 0 -# - qat_dev: crypto02 -# qat_id: "0000:xy:00.0" -# qat_module_type: qat_c62x -# qat_pci_type: c6xx -# qat_sriov_numvfs: 10 - -# - qat_dev: crypto03 -# qat_id: "0000:yz:00.0" -# qat_module_type: qat_c62x -# qat_pci_type: c6xx -# qat_sriov_numvfs: 10 + +# Install and configure OpenSSL cryptography +openssl_install: {{ bmra.features.qat.update_drivers }} # This requires update_qat_drivers set to 'true' in host vars # Enables hugepages support hugepages_enabled: {{ bmra.features.hugepages.enable }} @@ -113,9 +98,8 @@ hugepages_enabled: {{ bmra.features.hugepages.enable }} # Hugepage sizes available: 2M, 1G default_hugepage_size: {{ bmra.features.hugepages.default }} -# Sets how many hugepages of each size should be created -hugepages_1G: {{ bmra.features.hugepages.hugepages_1G }} -hugepages_2M: {{ bmra.features.hugepages.hugepages_2M }} +# Sets how many hugepages of 'default_hugepage_size' size should be created +number_of_hugepages: {{ bmra.features.hugepages.amount }} # CPU isolation from Linux scheduler isolcpus_enabled: {{ bmra.features.isolcpus.enable }} @@ -124,17 +108,18 @@ isolcpus_enabled: {{ bmra.features.isolcpus.enable }} isolcpus: "{{ bmra.features.isolcpus.cpus }}" # Native CPU Manager (Kubernetes built-in) - +# These settings are relevant only if in group_vars native_cpu_manager_enabled: true +# Amount of CPU cores that will be reserved for the housekeeping (2000m = 2000 millicores = 2 cores) native_cpu_manager_system_reserved_cpus: 2000m # Amount of CPU cores that will be reserved for Kubelet native_cpu_manager_kube_reserved_cpus: 1000m # Explicit list of the CPUs reserved from pods scheduling. # Note: Supported only with kube_version 1.17 and newer, overrides native_cpu_manager_system_reserved_cpus and native_cpu_manager_kube_reserved_cpus. #native_cpu_manager_reserved_cpus: "0,1,2" -# Note: All reamining unreserved CPU cores will be consumed by the workloads. +# Note: All remaining unreserved CPU cores will be consumed by the workloads. # Enable/Disable Intel PState scaling driver -intel_pstate_enabled: true +intel_pstate_enabled: false # Config options for intel_pstate: disable, passive, force, no_hwp, hwp_only, support_acpi_ppc, per_cpu_perf_limits intel_pstate: hwp_only # Enable/Disable Intel Turbo Boost PState attribute @@ -190,6 +175,10 @@ sst_cp_cpu_clos: - clos: 1 cpus: 3,7 +# Intel(R) SST-TF (feature turbo-freq) configuration for Ice Lake (ICX) Platforms. +# [true] Enable Intel Speed Select Turbo Frequency (SST-TF) +# [false] Disable Intel Speed Select Base Frequency (SST-TF) +sst_tf_configuration_enabled: false # (CentOS 7 only) install real time kernel and related packages for flexran install_real_time_package: false diff --git a/sw_config/bmra/Dockerfile b/sw_config/bmra/Dockerfile index 3f21241..d464822 100644 --- a/sw_config/bmra/Dockerfile +++ b/sw_config/bmra/Dockerfile @@ -10,7 +10,7 @@ RUN yum -y update && \ yum -y install git epel-release python36 python-netaddr && \ yum -y install python-pip && \ pip install --no-cache-dir pip==9.0.3 && \ - pip install --no-cache-dir ansible==2.9.17 jmespath && \ + pip install --no-cache-dir ansible==2.9.20 jmespath && \ pip install --no-cache-dir jinja2 --upgrade CMD ["bash"] diff --git a/sw_config/bmra/patched_cmk_build.yml b/sw_config/bmra/patched_cmk_build.yml index 0b5c774..a424c55 100644 --- a/sw_config/bmra/patched_cmk_build.yml +++ b/sw_config/bmra/patched_cmk_build.yml @@ -1,37 +1,12 @@ -# SPDX-FileCopyrightText: 2020 Intel Corporation. +# SPDX-FileCopyrightText: 2021 Intel Corporation. # # SPDX-License-Identifier: Apache-2.0 --- -- name: install epel-release on Red Hat based OS - package: name=epel-release - when: ansible_os_family == 'RedHat' - -# note: on Ubuntu, pip is installed via install_dependencies -- name: install pip - package: - name: python-pip - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version < '8' - -- name: install pip - package: - name: python3-pip - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version >= '8' - - name: install dependencies include_role: name: install_dependencies -- name: install Python dependencies - pip: - name: - - setuptools - - docker - - name: clone CMK repository git: repo: "{{ cmk_git_url }}" @@ -61,25 +36,42 @@ - name: build CMK image make: chdir: "{{ cmk_dir }}" + when: container_runtime == "docker" # NOTE(przemeklal): this fixes problem in CMK with ImagePullPolicy hardcoded to Never and the pod is scheduled on controller node - name: tag CMK image command: docker tag cmk:{{ cmk_img_version }} {{ registry_local_address }}/cmk:{{ cmk_img_version }} changed_when: true + when: container_runtime == "docker" - name: push CMK image to local registry command: docker push {{ registry_local_address }}/cmk:{{ cmk_img_version }} + changed_when: true when: + - container_runtime == "docker" - inventory_hostname == groups['kube-node'][0] + +- name: build and tag CMK image + command: podman build -f Dockerfile -t {{ registry_local_address }}/cmk:{{ cmk_img_version }} + args: + chdir: "{{ cmk_dir }}" changed_when: true + when: '"docker" not in container_runtime' + +- name: push CMK image to local registry + command: podman push {{ registry_local_address }}/cmk:{{ cmk_img_version }} + changed_when: true + when: + - inventory_hostname == groups['kube-node'][0] + - '"docker" not in container_runtime' -- name: clean up any preexisting certs/key/CSR files +- name: clean up any pre-existing certs/key/CSR files file: path=/etc/ssl/cmk state=absent when: inventory_hostname == groups['kube-master'][0] failed_when: false become: yes -- name: delete any preexisting certs/key/CSR from Kubernetes +- name: delete any pre-existing certs/key/CSR from Kubernetes command: kubectl delete csr cmk-webhook-{{ item }}.{{ cmk_namespace }} when: inventory_hostname == groups['kube-master'][0] failed_when: false @@ -212,7 +204,7 @@ when: - inventory_hostname == groups['kube-master'][0] -- name: get approved server certificate +- name: get approved server certificate shell: kubectl get csr cmk-webhook-server.{{ cmk_namespace }} -o jsonpath='{.status.certificate}' args: chdir: "/etc/ssl/cmk/" @@ -272,8 +264,13 @@ - name: restart kube-apiserver after updating admission control configuration when: inventory_hostname == groups['kube-master'][0] block: - - name: remove kube-apiserver Docker container - shell: docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f + - name: remove kube-apiserver container + # noqa 305 - shell is used intentionally here + shell: >- + {{ (container_runtime == 'docker') | ternary('docker ps -af name=k8s_kube-apiserver* -q | + xargs --no-run-if-empty docker rm -f', + 'crictl ps -a --name=kube-apiserver* -q | + xargs --no-run-if-empty crictl rm -f') }} args: executable: /bin/bash register: remove_apiserver_container @@ -345,11 +342,11 @@ when: - inventory_hostname == groups['kube-master'][0] -# remove any preexisting configmaps before cmk redeployment -- name: remove any preexisting configmaps before CMK deployment +# remove any pre-existing configmaps before cmk redeployment +- name: remove any pre-existing configmaps before CMK deployment command: kubectl delete cm cmk-config-{{ inventory_hostname }} when: - - inventory_hostname in cmk_hosts_list.split(',') + - inventory_hostname in (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) delegate_to: "{{ groups['kube-master']|first }}" failed_when: false diff --git a/sw_config/bmra/patched_k8s.yml b/sw_config/bmra/patched_k8s.yml index 5dfc3bd..fb0d43a 100644 --- a/sw_config/bmra/patched_k8s.yml +++ b/sw_config/bmra/patched_k8s.yml @@ -30,6 +30,10 @@ {%- endif -%} enable_admission_plugins_prepare: >- [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}] + bmra_docker_version: >- + {% if ansible_distribution_version >= '21.04' %}latest{% else %}19.03{%endif %} + flannel_backend_type: >- + {% if ansible_distribution_version >= '21.04' %}host-gw{% else %}vxlan{%endif %} kube_config_dir: /etc/kubernetes - name: set kube_cert_dir set_fact: @@ -38,15 +42,32 @@ environment: "{{ proxy_env | d({}) }}" any_errors_fatal: true +- hosts: all + tasks: + - name: add docker runtime vars + set_fact: + container_manager: docker + docker_iptables_enabled: true + docker_dns_servers_strict: false + docker_version: "{{ bmra_docker_version }}" + when: container_runtime == "docker" + - name: add containerd runtime vars + set_fact: + container_manager: containerd + etcd_deployment_type: host + containerd_extra_args: |2 + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry_local_address }}"] + endpoint = ["https://{{ registry_local_address }}"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry_local_address }}".tls] + ca_file = "/etc/containers/certs.d/{{ registry_local_address }}/ca.crt" + when: container_runtime == "containerd" - name: run kubespray import_playbook: kubespray/cluster.yml vars: kubeadm_enabled: true multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf - docker_iptables_enabled: true - docker_dns_servers_strict: false + nginx_image_tag: 1.21.1 override_system_hostname: false - docker_version: '19.03' kube_proxy_mode: iptables enable_nodelocaldns: false system_reserved: true @@ -105,6 +126,7 @@ - name: restart docker daemon to recreate iptables rules systemd: name=docker state=restarted become: yes + when: container_runtime == "docker" - name: restart kubelet to trigger static pods recreation systemd: name=kubelet state=restarted become: yes @@ -144,7 +166,7 @@ roles: - role: cluster_defaults tags: defaults - - role: docker_registry + - role: container_registry tags: registry - role: dockerhub_credentials when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file" diff --git a/sw_config/bmra/patched_kubespray_requirements.txt b/sw_config/bmra/patched_kubespray_requirements.txt index b6cf112..cceb6ff 100644 --- a/sw_config/bmra/patched_kubespray_requirements.txt +++ b/sw_config/bmra/patched_kubespray_requirements.txt @@ -2,10 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -ansible==2.9.17 -jinja2==2.11.1 +ansible==2.9.20 +cryptography==2.8 +jinja2==2.11.3 netaddr==0.7.19 pbr==5.4.4 jmespath==0.9.5 ruamel.yaml==0.16.10 +MarkupSafe==1.1.1 ruamel.yaml.clib==0.2.2 diff --git a/sw_config/bmra/patched_packages.yml b/sw_config/bmra/patched_packages.yml index 90a8519..14d7291 100644 --- a/sw_config/bmra/patched_packages.yml +++ b/sw_config/bmra/patched_packages.yml @@ -19,7 +19,7 @@ - setuptools<=44 extra_args: --upgrade -#pinned python package versions +#pinned python packages versions - name: install Python packages pip: name: @@ -27,7 +27,12 @@ - ruamel.yaml.clib==0.2.2 - ruamel.yaml==0.16.13 - cachetools=={{ (ansible_os_family == 'RedHat' and ansible_distribution_version < '8') | ternary('3.1.1', '4.2.1') }} + - markupsafe==1.1.1 + - jinja2==2.11.3 - openshift==0.11.2 + - six>=1.15.0 + - websocket-client==0.58.0 + - oauthlib==3.1.0 state: present register: pip_result retries: 5 diff --git a/sw_config/bmra/patched_preflight.yml b/sw_config/bmra/patched_preflight.yml new file mode 100644 index 0000000..41b7efd --- /dev/null +++ b/sw_config/bmra/patched_preflight.yml @@ -0,0 +1,544 @@ +# SPDX-FileCopyrightText: 2021 Intel Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +--- +# Preflight: ALL checks must PASS +# Only assert issues (do NOT change anything) +# +# Manual run: 'ansible-playbook -i inventory.ini playbooks/preflight.yml --flush-cache' +# +# Summary: +# On Ansible Host (localhost): +# - Check Ansible version (match) +# - Check Python version (min) +# - Check Group Vars (exist) +# - Check CMK Hosts (valid targets) +# On All targets (k8s-cluster): +# - Check Linux Distro +# - Check Hostnames (match Inventory) +# - Check CMK Config (isolcpus defined) +# - Check isolcpus Total (not more than actual) +# - Check isolcpus IDs (valid on system) +# - Check isolcpus OS Reserved (not 0,1,etc) +# On Worker Nodes Only (kube-node): +# - Check DP Interfaces (is not empty) +# - Check DP Interfaces Name (optional) +# - Check DP Interfaces Bus Info (pciid) +# - Check QAT Devices Bus Info (pciid) +# - Check QAT SRIOV VFs (max) +# - Check SGX configuration +# - Check OVS DPDK Dependencies (for 1G Hugepages) +# - Check VPP Dependencies (for 2M Hugepages) +# - Check CNI Dependencies (for OVS DPDK or VPP and Hugepages) +# - Check SST (not on RHEL 8.2 or old OSs) +# - Warn BIOS VT-d (should be enabled) +# - Warn BIOS Hyper-Threading (should be enabled) +# - Warn collectd (kernel update needed on old OSs) +# - Check OVS DPDK Version Compatability (for OVS support) + +# additional vars required: +# bmra_supported_ansible: # must be version +# bmra_supported_python: # min version +# bmra_supported_distros: [] # list +# bmra_supported_distros_versions: [] # list + + +################################## +# Prerequisites for Ansible Host # +################################## +- hosts: 127.0.0.1 + connection: local + vars: + bmra_supported_ansible: 2.9.20 + bmra_supported_python: 2.7 + + tasks: + + - debug: msg="Ansible version is {{ ansible_version.string }}" + - name: Check Ansible Version + assert: + that: (ansible_version.full is version_compare(bmra_supported_ansible, '==')) + msg: "Ansible version must be {{ bmra_supported_ansible }}. Please update" + + - debug: msg="Python version is {{ ansible_python_version }}" + - name: Check Python Version + assert: + that: (ansible_python_version is version_compare(bmra_supported_python, '>=')) + msg: "Python version must be at least {{ bmra_supported_python }}. Please update" + + - name: Read Group Vars + stat: + path: "{{ inventory_dir }}/group_vars/" + register: group_vars_details + + - name: Check Group Vars + assert: + that: "group_vars_details.stat.exists and group_vars_details.stat.isdir" + msg: "File group_vars/all.yml does NOT exist. Must be created per Guide" + + - debug: + msg: + - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml) + - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml) + - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml) + - all targets = {{ groups['all'] }} (inventory.ini) + when: cmk_enabled is defined # CMK expected true for all profiles except basic + + - name: Check Intel CMK Hosts + assert: + that: "item in groups['all']" + msg: "Hostname '{{ item }}' is NOT a valid target from inventory. Please correct the cmk_hosts_list or disable the CMK feature in group vars" + with_items: "{{ (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) }}" + when: cmk_enabled is defined and cmk_enabled and not cmk_use_all_hosts + + +############################################## +# Prerequisites for Control and Worker Nodes # +############################################## +- hosts: k8s-cluster + vars: + bmra_supported_distros: [CentOS, RedHat, Ubuntu] + bmra_supported_distros_versions: ['7.6', '7.8', '7.9', '8.2', '8.3', '8.4', '18.04', '20.04', '21.04'] + isolcpus_ranges: [] + isolcpus_discretes: [] + + tasks: + + - debug: msg="Linux distribution on target is {{ ansible_distribution }} {{ ansible_distribution_version }} {{ ansible_distribution_release }}" + - name: Check Linux Distro and Version + assert: + that: "ansible_distribution in bmra_supported_distros and ansible_distribution_version in bmra_supported_distros_versions" + msg: + - Linux distribution {{ ansible_distribution }} {{ ansible_distribution_version }} on target '{{ inventory_hostname }}' is NOT supported + - Must be one of {{ bmra_supported_distros }} and version {{ bmra_supported_distros_versions }} + +# - name: Check Linux Across Cluster +# TODO ?? Linux OS must be the same on all targets (no mix-n-match) + + - name: regather network facts in case hostname recently changed + setup: + gather_subset: network + - debug: msg="Inventory target '{{ inventory_hostname }}' has the actual system hostname '{{ ansible_hostname }}'" + - name: Check Inventory Hostnames + debug: + msg: + - "Target '{{ inventory_hostname }}' in inventory does NOT match the actual system hostname '{{ ansible_hostname }}'." + - "If it's done intentionally, please ignore this message." + when: + - inventory_hostname != ansible_hostname + +# Early check if SELinux is configured properly + - block: + - name: "Collect packages facts" + package_facts: + - debug: + msg: + - "Current SELinux status:" + - "status: {{ ansible_selinux.status | default('') }}" + - "policy version: {{ ansible_selinux.policyvers | default('') }}" + - "type: {{ ansible_selinux.type | default('') }}" + - "mode: {{ ansible_selinux.mode | default('') }}" + - "config_mode: {{ ansible_selinux.config_mode | default('') }}" + + - name: check selinux condition possibly causing system boot failure + debug: + msg: + - "Current SELinux setup might cause the system possibly will not boot up on next reboot." + - "Please, check SELinux settings and set it up according to the documentation." + when: + - "'selinux-policy' not in ansible_facts.packages" + - "'selinux-policy-targeted' not in ansible_facts.packages" + when: + - ansible_os_family == "RedHat" + +# STORY: "cmk requires isolcpus to be configured" + - block: + - debug: + msg: + - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml) + - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml) + - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml) + - cmk_shared_num_cores = {{ cmk_shared_num_cores }} (group_vars/all.yml) + - cmk_exclusive_num_cores = {{ cmk_exclusive_num_cores }} (group_vars/all.yml) + - isolcpus_enabled = {{ isolcpus_enabled }} (host_vars) + - isolcpus = {{ isolcpus }} (host_vars) + - ansible_processor_count = {{ ansible_processor_count }} + - ansible_processor_cores = {{ ansible_processor_cores }} + - ansible_processor_threads_per_core = {{ ansible_processor_threads_per_core }} + - ansible_processor_vcpus = {{ ansible_processor_vcpus }} + - CPUs Reserved for OS = 0...{{ ansible_processor_count - 1 }} +# - CPUs Reserved for OS = {{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }} # [E207] Nested jinja pattern + + - name: Check Intel CMK Config + assert: + that: ({{ cmk_enabled }} and {{ isolcpus_enabled }} and "{{ isolcpus }}" | length > 0) + msg: + - Incorrect configuration pertaining Intel CMK. Conflicting or improper values detected + - When Intel CMK is enabled, CPUs isolation ('isolcpus') must be set according to the example file for host_vars. Please correct the configuration + + - name: Split isolcpus Groups + set_fact: + isolcpus_groups: "{{ isolcpus.split(',') }}" + + - debug: msg="isolcpus_groups = {{ isolcpus_groups }}" + + - name: Filter isolcpus Ranges + set_fact: + isolcpus_ranges: "{{ isolcpus_ranges + [item] }}" + with_items: "{{ isolcpus_groups }}" + when: ("-" in item) + + - debug: msg="isolcpus_ranges = {{ isolcpus_ranges }}" + + - name: Filter isolcpus Discretes + set_fact: + isolcpus_discretes: "{{ isolcpus_discretes + [item] }}" + with_items: "{{ isolcpus_groups }}" + when: ("-" not in item) + + - debug: msg="isolcpus_discretes = {{ isolcpus_discretes }}" + + - name: Build isolcpus List + set_fact: + isolcpus_list: "{{ isolcpus_list | default([]) | union(isolcpus_discretes) | union([item]) }}" + with_sequence: "{{ isolcpus_ranges }}" + + - debug: msg="isolcpus_list = {{ isolcpus_list }}" + + - name: Check isolcpus Total + assert: + that: "{{ isolcpus_list | length }} <= ansible_processor_vcpus" + msg: + - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected + - The number of isolcpus {{ isolcpus_list | length }}, exceeds total CPUs on target {{ ansible_processor_vcpus }}. Please correct the configuration + when: isolcpus is defined + + - name: Check isolcpus IDs + assert: + that: "item | int <= ansible_processor_vcpus" + msg: + - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected + - The CPU ID {{ item }} set for isolcpus is NOT actually present on target. Please correct the configuration + with_items: "{{ isolcpus_list }}" + when: isolcpus is defined + + - name: Check isolcpus OS Reserved + assert: + that: "item not in isolcpus_list" + msg: + - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected + - The CPU ID 0...{{ ansible_processor_count - 1 }} should NOT be set for isolcpus. Please correct the configuration + with_items: "{{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }}" + when: isolcpus is defined + +#TODO relationship between cmk shared/exclusive cores and isolcpus + + when: + - cmk_enabled is defined + - (not cmk_hosts_list is defined) or (inventory_hostname in cmk_hosts_list) #CMK expected true for all profiles except basic + # {% if not cmk_use_all_hosts %} + - "'kube-node' in group_names" + # {% endif %} + + +#################################### +# Prerequisites for Worker Node(s) # +#################################### +- hosts: kube-node + vars: + phy_nics_pciids: [] + + tasks: + +# STORY: "nic bus info specified is present on system" + - debug: + msg: "Dataplane (DP) interface(s) defined in host_vars = {{ dataplane_interfaces }}" + when: dataplane_interfaces is defined + + - name: Check DP Interfaces + assert: + that: "dataplane_interfaces != []" + msg: "Dataplane (DP) interface(s) on target '{{ ansible_hostname }}' must be set in host_vars. Please correct the configuration" + when: + - dataplane_interfaces is defined + - (update_nic_drivers is defined and update_nic_drivers) or + (install_ddp_packages is defined and install_ddp_packages) or + (sriov_cni_enabled is defined and sriov_cni_enabled) or + (sriov_network_operator_enabled is defined and sriov_network_operator_enabled) + + - debug: + msg: "Network interfaces present on target '{{ ansible_hostname }}' = {{ ansible_interfaces }}" + + - name: Read Physical NICs PCIIDs + set_fact: + phy_nics_pciids: "{{ phy_nics_pciids + [ ansible_facts[item]['pciid'] ] }}" + with_items: "{{ ansible_interfaces }}" + when: ansible_facts[item]['pciid'] is defined and ansible_facts[item]['type'] == "ether" + + - debug: msg="PCI Slots for the NICs on target '{{ ansible_hostname }}' = {{ phy_nics_pciids }}" + + - name: Check DP Interfaces Names + assert: + that: ("{{ item.name }}" in {{ ansible_interfaces }}) + msg: "Dataplane interface '{{ item.name }}' defined in host_vars does NOT exist on target. Please correct the configuration" + with_items: "{{ dataplane_interfaces }}" + when: dataplane_interfaces is defined and dataplane_interfaces != [] + ignore_errors: True + + - name: Check DP Interfaces Bus Info + assert: + that: ("{{ item.bus_info }}" in "{{ phy_nics_pciids }}") + msg: "Dataplane interface '{{ item.name }}' defined with PCI ID '{{ item.bus_info }}' does NOT exist on target. Please correct the configuration" + with_items: "{{ dataplane_interfaces }}" + when: dataplane_interfaces is defined and dataplane_interfaces != [] + ignore_errors: True + + +# QAT Devices list is okay to be left empty (default), but if was defined, device(s) must exist on target + - debug: + msg: "QAT device(s) defined in host_vars = {{ qat_devices }}" + when: qat_devices is defined + + - name: Read QAT PCIIDs + shell: lshw -businfo -numeric | grep -i quickassist + register: lshw_qat + ignore_errors: True + when: qat_devices is defined + + - debug: + msg: "QAT devices found on target = {{ lshw_qat.stdout }}" + when: qat_devices is defined + + - name: Check QAT Devices' Bus Info + assert: + that: ("{{ item.qat_id }}" in """{{ lshw_qat.stdout }}""") + msg: "QAT device '{{ item.qat_dev }}' defined with PCI ID '{{ item.qat_id }}' does NOT exist on target. Please correct the configuration" + with_items: "{{ qat_devices }}" + when: qat_devices is defined and qat_devices != [] + ignore_errors: True + +# STORY: "qat_sriov_numvfs should not exceed max supported (16) per each dev_ID" + - debug: + msg: + - qat_sriov_numvfs for {{ item.qat_id }} = {{ item.qat_sriov_numvfs }} (host_vars) + - update_qat_drivers = {{ update_qat_drivers }} (host_vars) + with_items: "{{ qat_devices }}" + when: qat_devices is defined and qat_devices != [] # update_qat_drivers expected as 'true' for all profiles except basic + + - name: Check QAT SRIOV VFs + assert: + that: ({{ item.qat_sriov_numvfs }} <= 16) + msg: + - Incorrect configuration pertaining QAT SRIOV. Conflicting or improper values detected + - When SRIOV VFs are set for QAT, max value is 16 for each ID (max 48 total per card). Please correct the configuration + with_items: "{{ qat_devices }}" + when: + - update_qat_drivers is defined and update_qat_drivers + - qat_devices is defined and qat_devices != [] +# OpenSSL & OpenSSL*Engine must only be configured / installed when update_qat_drivers is set to 'true' and qat_devices is defined in host vars + - name: check OpenSSL and OpenSSL*Engine requirements + assert: + that: + - update_qat_drivers + - qat_devices is defined and qat_devices != [] + fail_msg: "OpenSSL & OpenSSL*Engine will only configured if update_qat_drivers is set to 'true' & qat_devices is defined in host vars" + success_msg: "OpenSSL & OpenSSL*Engine verification completed" + when: openssl_install is defined and openssl_install + + - name: check KMRA requirements + assert: + that: + - sgx_dp_enabled + fail_msg: "KMRA installation requires sgx_dp_enabled set to 'true'" + success_msg: "KMRA requirements verified" + when: kmra_enabled is defined and kmra_enabled + + - name: check SGX configuration + assert: + that: + - sgx_enabled + fail_msg: "SGX drivers installation requires sgx_enabled set to 'true'" + success_msg: "SGX configuration verified" + when: + - sgx_dp_enabled is defined and sgx_dp_enabled + - (ansible_distribution == 'Ubuntu' and ansible_distribution_version != '21.04') + or (ansible_os_family == 'RedHat' and ansible_distribution_version != '8.4') + + - name: check NFD configuration + assert: + that: + - nfd_enabled + fail_msg: "SGX DP requires nfd_enabled set to 'true'" + success_msg: "NFD configuration verified" + when: sgx_dp_enabled is defined and sgx_dp_enabled + + - name: check kmra_pccs_api_key presence + assert: + that: + - kmra_pccs_api_key is defined + fail_msg: + - "kmra_pccs_api_key is not defined" + success_msg: "kmra_pccs_api_key presence is verified" + when: + - kmra_enabled is defined and kmra_enabled + + - name: check PCCS API key length + assert: + that: + - kmra_pccs_api_key | length == 32 + fail_msg: "PCCS API Key should be 32 bytes long" + success_msg: "PCCS API key length verified" + when: + - kmra_enabled is defined and kmra_enabled + + - name: check PCCS API key is not a placeholder + assert: + that: + - kmra_pccs_api_key is defined + - kmra_pccs_api_key != "ffffffffffffffffffffffffffffffff" + fail_msg: + - "Please, visit https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'" + - "to generate PCCS API key." + - "PCCS API key is essential for KMRA deployment and usage." + success_msg: "PCCS API key verified" + when: + - kmra_enabled is defined and kmra_enabled + +# STORY: "vpp/ovsdpdk require hugepage enabled and configured" + - debug: + msg: + - vpp_enabled = {{ vpp_enabled }} (host_vars) + - example_net_attach_defs = {{ example_net_attach_defs }} (group_vars/all.yml) + - userspace_ovs_dpdk = {{ example_net_attach_defs['userspace_ovs_dpdk'] }} (group_vars/all.yml) + - userspace_vpp = {{ example_net_attach_defs['userspace_vpp'] }} (group_vars/all.yml) + - sriov_net_dp = {{ example_net_attach_defs['sriov_net_dp'] }} (group_vars/all.yml) + - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars) + - sriov_cni_enabled = {{ sriov_cni_enabled }} (host_vars) + - sriov_network_operator_enabled = {{ sriov_network_operator_enabled }} (host_vars) + - bond_cni_enabled = {{ bond_cni_enabled }} (host_vars) + - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars) + - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars) + - hugepages_enabled = {{ hugepages_enabled }} (host_vars) + - default_hugepage_size = {{ default_hugepage_size }} (host_vars) + - number_of_hugepages = {{ number_of_hugepages }} (host_vars) + when: vpp_enabled is defined #host_vars + + - name: Check OVS DPDK Dependencies + assert: + that: >- + ({{ ovs_dpdk_enabled }} and not {{ vpp_enabled }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0) + or {{ vpp_enabled }} + msg: + - Incorrect configuration pertaining OVS DPDK. Conflicting or improper values detected + - When OVS DPDK is enabled, VPP must be disabled and Hugepages must be set to 1G according to host_vars example. Please correct the configuration + when: ovs_dpdk_enabled is defined and ovs_dpdk_enabled + + - name: Check VPP Dependencies + assert: + that: >- + ({{ vpp_enabled }} and not {{ ovs_dpdk_enabled }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0) + or {{ ovs_dpdk_enabled }} + msg: + - Incorrect configuration pertaining VPP. Conflicting or improper values detected + - When VPP is enabled, OVS DPDK must be disabled and Hugepages must be set to 2M according to host_vars example. Please correct the configuration + when: vpp_enabled is defined and vpp_enabled + + +# STORY: "cnis require net-attach-defs to be enabled" + - name: Check CNI Config + assert: + that: >- + ({{ userspace_cni_enabled }} and {{ ovs_dpdk_enabled }} and {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and not {{ vpp_enabled }} and + not {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0) + or ({{ userspace_cni_enabled }} and not {{ ovs_dpdk_enabled }} and not {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and {{ vpp_enabled }} + and {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0) + msg: + - Incorrect configuration pertaining CNI. Conflicting or improper values detected. + - When CNI is enabled, either OVS DPDK either VPP must be enabled and Hugepages must be according to example files. Please correct the configuration + when: userspace_cni_enabled is defined and userspace_cni_enabled + + +# STORY: "If SST enabled, confirm minimum kernel or kernel_update specified" + - name: Check SST # see Jira NPF-1545 + assert: + that: (not sst_bf_configuration_enabled) + msg: "SST-BF is NOT supported on {{ ansible_distribution }} {{ ansible_distribution_version }}. Please use a different OS or disable this feature" + when: + - sst_bf_configuration_enabled is defined + - (ansible_distribution == "RedHat" and ansible_distribution_version == '8.2') or ansible_distribution_version in ['7.6', '7.8', '7.9', '18.04'] + ignore_errors: True + +# STORY: Intel VT-d should be enabled in BIOS + - name: Check Intel VT-d + shell: dmesg | grep DMAR | grep remapping + register: dmesg_dmar_remap + ignore_errors: True + changed_when: False + + - debug: msg="dmesg >> {{ dmesg_dmar_remap.stdout }}" + + - name: Warn about Intel VT-d + fail: + msg: "Warning: Intel VT-d appears DISABLED on target. Please check BIOS under 'Advanced > Integrated IO Configuration' and Enable if necessary" + when: dmesg_dmar_remap.stdout|length == 0 + ignore_errors: True + + +# STORY: CPU Hyper-Threading should be enabled in BIOS + - name: Warn about Hyper-Threading + fail: + msg: "Warning: CPU Hyper-Threading is DISABLED on target. Please check BIOS under 'Advanced > Processor Configuration' and Enable if necessary" + when: ansible_processor_threads_per_core != 2 + ignore_errors: True + + +# STORY: "check for collectd. See Jira NPF-1687" + - name: Warn about collectd + fail: + msg: "Warning: On {{ ansible_distribution }} {{ ansible_distribution_version }} collectd won't work unless 'update_kernel' is enabled in group_vars" + when: ansible_distribution_version in ['7.6', '18.04'] + ignore_errors: True + + +# STORY: TEMPORARY: "ovs dpdk version requirements" + - debug: + msg: + - install_dpdk = {{ install_dpdk }} (host_vars) + - dpdk_version = {{ dpdk_version }} (host_vars) + - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars) + - ovs_version = {{ ovs_version }} (host_vars) + when: + - install_dpdk is defined #host_vars + - dpdk_version is defined #host_vars + - ovs_version is defined #host_vars + - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars + + - name: Check OVS DPDK compatibility + assert: + that: >- + "{{ ovs_version }} == \"v2.15.0\" and {{ dpdk_version }} >= \"20.11\"" + or "{{ ovs_version }} == \"v2.14.2\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.14.1\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.14.0\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.3\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.2\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.1\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.0\" and {{ dpdk_version }} == \"19.11.6\"" + msg: "OVS {{ ovs_version }} does not build with DPDK version {{ dpdk_version }}. Please correct the host_vars configuration" + when: + - dpdk_version is defined #host_vars + - ovs_version is defined #host_vars + - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars + + + - meta: end_play + +# - name: Print all variables/facts known for a host +# ansible.builtin.debug: +# var: hostvars[inventory_hostname] +# verbosity: 4 diff --git a/sw_config/bmra/patched_rhel_packages.yml b/sw_config/bmra/patched_rhel_packages.yml index 40b22b6..687142a 100644 --- a/sw_config/bmra/patched_rhel_packages.yml +++ b/sw_config/bmra/patched_rhel_packages.yml @@ -10,7 +10,7 @@ - ansible_distribution == "CentOS" - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3' -- name: enable PowerTools repository on CentOS >= 8.3 +- name: enable powertools repository on CentOS >= 8.3 # noqa 303 - yum is called intenionallly here command: yum config-manager --set-enabled powertools when: @@ -88,7 +88,10 @@ package: name: "{{ item }}" state: present + retries: 5 + delay: 10 register: source_status + until: source_status is not failed with_items: - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-193.el8.x86_64.rpm" - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-193.el8.x86_64.rpm" @@ -103,22 +106,35 @@ state: present register: source_status with_items: - - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" - - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" -# - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" -# - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" + - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" + - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" when: - ansible_distribution == "CentOS" - ansible_distribution_version == '8.3' - not update_kernel +- name: pull matching kernel headers on CentOS 8.4 + package: + name: "{{ item }}" + state: present + register: source_status + with_items: + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.3.1.el8.x86_64.rpm" + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.3.1.el8.x86_64.rpm" +# - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.el8.x86_64.rpm" +# - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.el8.x86_64.rpm" + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version == '8.4' + - not update_kernel + # pull the matching kernel headers if kernel is not updated - name: pull matching kernel headers from configured repos # noqa 503 - more than one condition, can't be a handler package: name: - - kernel-headers-{{ ansible_kernel }} - - kernel-devel-{{ ansible_kernel }} + - kernel-headers-{{ ansible_kernel }} + - kernel-devel-{{ ansible_kernel }} register: kernel_source retries: 3 until: kernel_source is success @@ -196,7 +212,7 @@ - ansible_distribution in ["RedHat", "CentOS"] - ansible_distribution_version < '8' -- name: Set python is python3 +- name: set python is python3 alternatives: name: python path: /usr/bin/python3 @@ -214,6 +230,51 @@ state: present when: ansible_distribution in ["RedHat", "CentOS"] +- name: Add kubic yum repo and install updated version of podman + block: + - name: disable container-tools module + # noqa 305 - shell is used intentionally here + shell: dnf -y module disable container-tools + - name: enable rhcontainerbot/container-selinux repository + # noqa 305 - shell is used intentionally here + shell: dnf -y copr enable rhcontainerbot/container-selinux + - name: Add kubic yum repo + yum_repository: + name: devel_kubic_libcontainers_stable + description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever) + baseurl: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/ + gpgcheck: yes + gpgkey: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key + keepcache: '0' + - name: install runc + dnf: + name: runc + state: present + - name: install podman package + package: + name: podman + state: present + when: + - ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2' + - '"docker" not in container_runtime' + +# SELINUX will be disabled later stage so, these packages are required when container_runtime is docker +- name: install packages in RHEL >= 8.4 when container_runtime is docker + dnf: + name: + - bridge-utils + - lsof + - lvm2 + - tcpdump + - iproute-tc + - openssh-server + - chrony + - iputils + when: + - ansible_distribution == "RedHat" and ansible_distribution_version >= '8.4' + - container_runtime == "docker" + +# Workaround - Set pip to a version that supports correct version of packages needed - name: use the correct pip version for CentOS 7 pip: name: @@ -221,3 +282,11 @@ when: - ansible_distribution == "CentOS" - ansible_distribution_version < '7.9' + +- name: Install pip3 required by dpdk + yum: + name: python3-pip + state: latest + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version < '7.9' diff --git a/sw_config/bmra/patched_sriov_cni_install.yml b/sw_config/bmra/patched_sriov_cni_install.yml new file mode 100644 index 0000000..ab0dc52 --- /dev/null +++ b/sw_config/bmra/patched_sriov_cni_install.yml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2021 Intel Corporation. +# +# SPDX-License-Identifier: Apache-2.0 +--- +- name: install dependencies + include_role: + name: install_dependencies + +- name: clone sriov-cni repository + git: + repo: "{{ sriov_cni_url }}" + dest: "{{ sriov_cni_dir }}" + version: "{{ sriov_cni_version }}" + force: yes + +# Ignore errors as this has been failing +- name: build sriov-cni plugin + make: + chdir: /usr/src/sriov-cni + ignore_errors: true + +# missing go.sum entry for module providing package golang.org/x/sys/unix +- name: Force checkout as some files change after build error + shell: + cmd: git checkout -- . + chdir: /usr/src/sriov-cni + +- name: build sriov-cni plugin + make: + chdir: /usr/src/sriov-cni + +- name: create /opt/cni/bin + file: + path: "/opt/cni/bin" + state: directory + recurse: yes + mode: 0755 + +- name: install sriov-cni binary to /opt/cni/bin directory + copy: + src: /usr/src/sriov-cni/build/sriov + dest: /opt/cni/bin/sriov + mode: 0755 + remote_src: yes |