Updates Kuberef to use BMRA v21.08

Updates BMRA tag to v21.08
Updates Kuberef templates for BMRA configuration
Updates idf.yml (hw_config) to match changes to BMRA
Adds new and updated BMRA patches
Fixes a few bugs in functions.sh

Signed-off-by: Michael S. Pedersen <michaelx.pedersen@intel.com>
Change-Id: Ifbd4d60af76dac45e49145f00cbeb90de0f7c719
Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/72927
Tested-by: jenkins-ci <jenkins-opnfv-ci@opnfv.org>
Reviewed-by: Rihab Banday <rihab.banday@ericsson.com>

15 files changed, 818 insertions, 109 deletions

diff --git a/functions.sh b/functions.sh
index 69a3de7..89c1ebb 100755
--- a/functions.sh
+++ b/functions.sh
@@ -166,9 +166,9 @@ provision_hosts_baremetal() {
 if [ ! -d "${PROJECT_ROOT}/engine" ]; then
     ssh-keygen -t rsa -N "" -f "${PROJECT_ROOT}"/.ssh/id_rsa
     git clone https://gerrit.nordix.org/infra/engine.git
-    cp "${PROJECT_ROOT}"/"${VENDOR}"/{pdf.yaml,idf.yaml} \
-    "${PROJECT_ROOT}"/engine/engine
 fi
+cp "${PROJECT_ROOT}"/"${VENDOR}"/{pdf.yaml,idf.yaml} \
+"${PROJECT_ROOT}"/engine/engine
 cd "${PROJECT_ROOT}"/engine/engine || return
 ./deploy.sh -s ironic -d "${DISTRO}" \
 -p file:///"${PROJECT_ROOT}"/engine/engine/pdf.yaml \
@@ -181,9 +181,9 @@ provision_hosts_vms() {
 # Install and run cloud-infra
 if [ ! -d "$CURRENTPATH/engine" ]; then
     git clone https://gerrit.nordix.org/infra/engine.git "${CURRENTPATH}"/engine
-    cp "$CURRENTPATH"/hw_config/"$VENDOR"/{pdf.yaml,idf.yaml} \
-    "${CURRENTPATH}"/engine/engine
 fi
+cp "$CURRENTPATH"/hw_config/"$VENDOR"/{pdf.yaml,idf.yaml} \
+"${CURRENTPATH}"/engine/engine
 cd "$CURRENTPATH"/engine/engine || return
 ./deploy.sh -s ironic \
 -p file:///"${CURRENTPATH}"/engine/engine/pdf.yaml \
@@ -211,7 +211,7 @@ provision_k8s_baremetal() {
     ansible_cmd="/bin/bash -c '"
     if [[ "$DEPLOYMENT" == "k8s" ]]; then
         ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
-        ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.17; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
+        ansible_cmd+="yum -y remove python-netaddr; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
     fi
     ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'"
 
@@ -227,7 +227,7 @@ if ! command -v docker; then
     done
 fi
 if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
-    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 "${PROJECT_ROOT}"/container-experience-kits/
+    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${PROJECT_ROOT}"/container-experience-kits/
     cp -r "${PROJECT_ROOT}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${PROJECT_ROOT}"/container-experience-kits/
 fi
 if [ -f "${PROJECT_ROOT}/docker_config" ]; then
@@ -252,13 +252,17 @@ cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_packages.yml \
     "${PROJECT_ROOT}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
 cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_kubespray_requirements.txt \
     "${PROJECT_ROOT}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
+cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_preflight.yml \
+    "${PROJECT_ROOT}"/container-experience-kits/playbooks/preflight.yml
+cp "${PROJECT_ROOT}"/"${INSTALLER}"/patched_sriov_cni_install.yml \
+    "${PROJECT_ROOT}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
 
 sudo docker run --rm \
 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
 -e PROFILE="${BMRA_PROFILE}" \
 -v "${PROJECT_ROOT}"/container-experience-kits:/bmra \
--v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.03-install:centos \
-"${ansible_cmd}"
+-v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
+${ansible_cmd}
 EOF
 }
 
@@ -266,7 +270,7 @@ provision_k8s_vms() {
     # shellcheck disable=SC2087
 # Install BMRA
 if [ ! -d "${CURRENTPATH}/container-experience-kits" ]; then
-    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 "${CURRENTPATH}"/container-experience-kits/
+    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.08 "${CURRENTPATH}"/container-experience-kits/
     cp -r "${CURRENTPATH}"/container-experience-kits/examples/"${BMRA_PROFILE}"/group_vars "${CURRENTPATH}"/container-experience-kits/
 fi
 cp "${CURRENTPATH}"/sw_config/bmra/{inventory.ini,ansible.cfg} \
@@ -283,14 +287,19 @@ cp "${CURRENTPATH}"/sw_config/bmra/patched_packages.yml \
     "${CURRENTPATH}"/container-experience-kits/roles/bootstrap/install_packages/tasks/main.yml
 cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_kubespray_requirements.txt \
     "${CURRENTPATH}"/container-experience-kits/playbooks/k8s/kubespray/requirements.txt
+cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_preflight.yml \
+    "${CURRENTPATH}"/container-experience-kits/playbooks/preflight.yml
+cp "${CURRENTPATH}"/sw_config/"${INSTALLER}"/patched_sriov_cni_install.yml \
+    "${CURRENTPATH}"/container-experience-kits/roles/sriov_cni_install/tasks/main.yml
 
 ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
 
+# Ansible upgrade below can be removed once image is updated
 sudo docker run --rm \
 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
 -e PROFILE="${BMRA_PROFILE}" \
 -v "${CURRENTPATH}"/container-experience-kits:/bmra \
--v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.03-install:centos \
+-v ~/.ssh/:/root/.ssh/ rihabbanday/bmra21.08-install:centos \
 ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/"${BMRA_PROFILE}".yml
 }
 
diff --git a/hw_config/equinix-metal/idf.yaml b/hw_config/equinix-metal/idf.yaml
index 0cdcc41..a1f9ef8 100644
--- a/hw_config/equinix-metal/idf.yaml
+++ b/hw_config/equinix-metal/idf.yaml
@@ -52,9 +52,9 @@ bmra:
 #    qat:
 #      - name: crypto01
 #        pci: "0000:ab:00.0"
-#        mod_type: qat_c62x
 #        pci_type: c6xx
 #        vfs: 4
+  runtime: docker # 'docker' and 'containerd' supported
   features:
     sriov:
       enable: true
@@ -65,8 +65,7 @@ bmra:
     hugepages:
       enable: true
       default: 2M
-      hugepages_1G: 0
-      hugepages_2M: 10240
+      amount: 10240
     isolcpus:
       enable: true
       autogenerate: true
diff --git a/hw_config/ericsson-pod1/idf.yaml b/hw_config/ericsson-pod1/idf.yaml
index 411d36d..3e03d4c 100644
--- a/hw_config/ericsson-pod1/idf.yaml
+++ b/hw_config/ericsson-pod1/idf.yaml
@@ -81,9 +81,9 @@ bmra:
 #    qat:
 #      - name: crypto01
 #        pci: "0000:ab:00.0"
-#        mod_type: qat_c62x
 #        pci_type: c6xx
 #        vfs: 4
+  runtime: docker # 'docker' and 'containerd' supported
   features:
     sriov:
       enable: false
@@ -94,8 +94,7 @@ bmra:
     hugepages:
       enable: false
       default: 2M
-      hugepages_1G: 0
-      hugepages_2M: 10240
+      amount: 10240
     isolcpus:
       enable: true
       autogenerate: true
diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml
index 72f8c95..dc95638 100644
--- a/hw_config/intel/idf.yaml
+++ b/hw_config/intel/idf.yaml
@@ -81,9 +81,9 @@ bmra:
 #    qat:
 #      - name: crypto01
 #        pci: "0000:ab:00.0"
-#        mod_type: qat_c62x
 #        pci_type: c6xx
 #        vfs: 4
+  runtime: docker # 'docker' and 'containerd' supported
   features:
     sriov:
       enable: true
@@ -94,8 +94,7 @@ bmra:
     hugepages:
       enable: true
       default: 2M
-      hugepages_1G: 0
-      hugepages_2M: 10240
+      amount: 10240
     isolcpus:
       enable: true
       autogenerate: true
diff --git a/hw_config/libvirt-vms/idf.yaml b/hw_config/libvirt-vms/idf.yaml
index 1616f9f..ab4fe0a 100644
--- a/hw_config/libvirt-vms/idf.yaml
+++ b/hw_config/libvirt-vms/idf.yaml
@@ -77,9 +77,9 @@ bmra:
 #    qat:
 #      - name: crypto01
 #        pci: "0000:ab:00.0"
-#        mod_type: qat_c62x
 #        pci_type: c6xx
 #        vfs: 4
+  runtime: docker # 'docker' and 'containerd' supported
   features:
     sriov:
       enable: false
@@ -90,12 +90,11 @@ bmra:
     hugepages:
       enable: true
       default: 2M
-      hugepages_1G: 0
-      hugepages_2M: 5120
+      amount: 5120
     isolcpus:
-      enable: false
+      enable: true
       autogenerate: false
-      cpus: "4-19,24-39,44-59,64-79" # Not used when autogenerate is true
+      cpus: "2-5" # Not used when autogenerate is true
     nfd: true
     cmk:
       enable: true
diff --git a/playbooks/roles/bmra-config/templates/all.j2 b/playbooks/roles/bmra-config/templates/all.j2
index 1eca556..440b373 100644
--- a/playbooks/roles/bmra-config/templates/all.j2
+++ b/playbooks/roles/bmra-config/templates/all.j2
@@ -9,9 +9,12 @@ SPDX-License-Identifier: Apache-2.0
 
 # Kubernetes version
 kubernetes: true
-#kube_version: v1.20.4
-kube_version: v1.19.8
-#kube_version: v1.18.16
+kube_version: v1.21.1
+#kube_version: v1.20.6
+#kube_version: v1.19.8
+
+# Kubernetes container runtime: docker, containerd
+container_runtime: {{ bmra.runtime }}
 
 # Run system-wide package update (apt dist-upgrade, yum update, ...)
 # Note: enabling this may lead to unexpected results
@@ -47,6 +50,10 @@ topology_manager_enabled: {{ bmra.features.topology_manager.enable }}
 # There are four supported policies: none, best-effort, restricted, single-numa-node.
 topology_manager_policy: "{{ bmra.features.topology_manager.policy }}"
 
+# OpenShift SRIOV Network Operator
+sriov_network_operator_enabled: false
+sriov_network_operator_namespace: "sriov-network-operator"
+
 # Intel SRIOV Network Device Plugin
 sriov_net_dp_enabled: {{ bmra.features.sriov_net_dp }}
 sriov_net_dp_namespace: kube-system
@@ -91,19 +98,26 @@ sriovdp_config_data: |
         ]
     }
 
+# Intel Device Plugin Operator
+intel_dp_namespace: kube-system # namespace will be applied for SGX DP and GPU DP
+
 # Intel QAT Device Plugin for Kubernetes
 qat_dp_enabled: {{ bmra.features.qat.enable }}
 qat_dp_namespace: kube-system
 qat_dp_build_image_locally: true
 
+# This feature will enable OpenSSL*Engine
+openssl_engine_enabled: false # To activate OpenSSL*Engine, install_openssl & update_qat_drivers must set to ‘true’ in host_vars
+
 # Intel GPU Device Plugin for Kubernetes
 gpu_dp_enabled: false
-gpu_dp_namespace: kube-system
+gpu_dp_kernel_version: "5.4.48+"
 gpu_dp_build_image_locally: true
 
 # Intel SGX Device Plugin for Kubernetes
 sgx_dp_enabled: false
 sgx_dp_build_image_locally: true
+sgx_aesmd_namespace: kube-system
 # ProvisionLimit is a number of containers that can share
 # the same SGX provision device.
 sgx_dp_provision_limit: 20
@@ -111,6 +125,16 @@ sgx_dp_provision_limit: 20
 # same SGX enclave device.
 sgx_dp_enclave_limit: 20
 
+# KMRA (Key Management Reference Application)
+kmra_enabled: false
+# The PCCS uses this API key to request collaterals from Intel's Provisioning Certificate Service.
+# User needs to subscribe first to obtain an API key.
+# For how to subscribe to Intel Provisioning Certificate Service and receive an API key,
+# goto https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'.
+kmra_pccs_api_key: "ffffffffffffffffffffffffffffffff"
+# deploy KMRA demo workload (NGINX server)
+kmra_deploy_demo_workload: true
+
 # Intel Telemetry Aware Scheduling
 tas_enabled: {{ bmra.features.tas.enable }}
 tas_namespace: monitoring
@@ -153,9 +177,17 @@ helm_enabled: true
 # local Docker Hub mirror, if it exists
 #docker_registry_mirrors:
 #  - http://mirror_ip:mirror_port
+#containerd_registries:
+#  "docker.io":
+#    - "https://registry-1.docker.io"
+#    - "https://mirror_ip:mirror_port"
 
 # Docker registry running on the cluster allows us to store images not avaialble on Docker Hub, e.g. CMK
-registry_local_address: "localhost:30500"
+# The range of valid ports is 30000-32767
+registry_nodeport: 30500
+{% raw %}
+registry_local_address: "localhost:{{ registry_nodeport }}"
+{% endraw %}
 
 # Enable Pod Security Policy. This option enables PSP admission controller and creates minimal set of rules.
 psp_enabled: {{ bmra.features.psp }}
diff --git a/playbooks/roles/bmra-config/templates/kube-node.j2 b/playbooks/roles/bmra-config/templates/kube-node.j2
index 2f66c5a..51c4112 100644
--- a/playbooks/roles/bmra-config/templates/kube-node.j2
+++ b/playbooks/roles/bmra-config/templates/kube-node.j2
@@ -40,22 +40,22 @@ bond_cni_enabled: {{ bmra.features.bond_cni }}
 install_dpdk: true
 
 # DPDK version
-dpdk_version: "19.11.6"
+dpdk_version: "21.05"
 
 # Custom DPDK patches local path
-# dpdk_local_patches_dir: "/tmp/patches/dpdk-19.11.6"
+# dpdk_local_patches_dir: "/tmp/patches/dpdk-20.11"
 
 # Userspace networking
 userspace_cni_enabled: false
-ovs_dpdk_enabled: false # Should be enabled with Userspace CNI, when VPP is set to "false"; 1G hugepages required
-ovs_version: "v2.13.0"
+ovs_dpdk_enabled: false # Should be enabled with Userspace CNI, when VPP is set to "false"; 1G default_hugepage_size required
+ovs_version: "v2.15.0"
 # CPU mask for OVS-DPDK PMD threads
 ovs_dpdk_lcore_mask: 0x1
 # Huge memory pages allocated by OVS-DPDK per NUMA node in megabytes
 # example 1: "256,512" will allocate 256MB from node 0 abd 512MB from node 1
-# example 2: "1024" will allocate 1GB fron node 0 on a single socket board, e.g. in a VM
+# example 2: "1024" will allocate 1GB from node 0 on a single socket board, e.g. in a VM
 ovs_dpdk_socket_mem: "256,0"
-vpp_enabled: false # Should be enabled with Userspace CNI, when ovs_dpdk is set to "false"; 2M hugepages required
+vpp_enabled: false # Should be enabled with Userspace CNI, when ovs_dpdk is set to "false"; 2M default_hugepage_size required
 
 # Set to 'true' to update i40e, ice and iavf kernel modules
 update_nic_drivers: false
@@ -82,30 +82,15 @@ qat_devices:
 {% for dev in bmra.device_roles.qat %}
   - qat_dev: {{ dev.name }}
     qat_id: "{{ dev.pci }}"
-    qat_module_type: {{ dev.mod_type }}
     qat_pci_type: {{ dev.pci_type }}
     qat_sriov_numvfs: {{ dev.vfs }}
 {% endfor %}
 {% else %}
 qat_devices: []
 {% endif %}
-#  - qat_dev: crypto01                # Device name as separate QAT Symmetric Crypto devices on which qat_sriov_numvfs will be distributed
-#    qat_id: "0000:ab:00.0"           # Device QAT id one using DPDK compatible driver for VF devices to be used by vfio-pci kernel driver, replace as required
-#    qat_module_type: qat_c62x        # QAT Crypto Poll Mode Kernel Module supported are qat_dh895xcc,qat_c62x,qat_c3xxx,qat_200xx,qat_c4xxx,qat_d15xx
-#    qat_pci_type: c6xx               # QAT Crypto Poll Mode Pci Driver id supported are dh895xcc,c6xx,c3xxx,d15xx,200xx & c4xxx
-#    qat_sriov_numvfs: 12             # set how many VFs per qat_id to be created such as c6xxvf support 32 so per device will be 10+10+12=32, replace as required
-                                     # Note: If don't want to assign VFs to id leave it as 0 such as qat_sriov_numvfs: 0
-#  - qat_dev: crypto02
-#    qat_id: "0000:xy:00.0"
-#    qat_module_type: qat_c62x
-#    qat_pci_type: c6xx
-#    qat_sriov_numvfs: 10
-
-#  - qat_dev: crypto03
-#    qat_id: "0000:yz:00.0"
-#    qat_module_type: qat_c62x
-#    qat_pci_type: c6xx
-#    qat_sriov_numvfs: 10
+
+# Install and configure OpenSSL cryptography
+openssl_install: {{ bmra.features.qat.update_drivers }} # This requires update_qat_drivers set to 'true' in host vars
 
 # Enables hugepages support
 hugepages_enabled: {{ bmra.features.hugepages.enable }}
@@ -113,9 +98,8 @@ hugepages_enabled: {{ bmra.features.hugepages.enable }}
 # Hugepage sizes available: 2M, 1G
 default_hugepage_size: {{ bmra.features.hugepages.default }}
 
-# Sets how many hugepages of each size should be created
-hugepages_1G: {{ bmra.features.hugepages.hugepages_1G }}
-hugepages_2M: {{ bmra.features.hugepages.hugepages_2M }}
+# Sets how many hugepages of 'default_hugepage_size' size should be created
+number_of_hugepages: {{ bmra.features.hugepages.amount }}
 
 # CPU isolation from Linux scheduler
 isolcpus_enabled: {{ bmra.features.isolcpus.enable }}
@@ -124,17 +108,18 @@ isolcpus_enabled: {{ bmra.features.isolcpus.enable }}
 isolcpus: "{{ bmra.features.isolcpus.cpus }}"
 
 # Native CPU Manager (Kubernetes built-in)
-
+# These settings are relevant only if in group_vars native_cpu_manager_enabled: true
+# Amount of CPU cores that will be reserved for the housekeeping (2000m = 2000 millicores = 2 cores)
 native_cpu_manager_system_reserved_cpus: 2000m
 # Amount of CPU cores that will be reserved for Kubelet
 native_cpu_manager_kube_reserved_cpus: 1000m
 # Explicit list of the CPUs reserved from pods scheduling.
 # Note: Supported only with kube_version 1.17 and newer, overrides native_cpu_manager_system_reserved_cpus and native_cpu_manager_kube_reserved_cpus.
 #native_cpu_manager_reserved_cpus: "0,1,2"
-# Note: All reamining unreserved CPU cores will be consumed by the workloads.
+# Note: All remaining unreserved CPU cores will be consumed by the workloads.
 
 # Enable/Disable Intel PState scaling driver
-intel_pstate_enabled: true
+intel_pstate_enabled: false
 # Config options for intel_pstate: disable, passive, force, no_hwp, hwp_only, support_acpi_ppc, per_cpu_perf_limits
 intel_pstate: hwp_only
 # Enable/Disable Intel Turbo Boost PState attribute
@@ -190,6 +175,10 @@ sst_cp_cpu_clos:
   - clos: 1
     cpus: 3,7
 
+# Intel(R) SST-TF (feature turbo-freq) configuration for Ice Lake (ICX) Platforms.
+# [true] Enable Intel Speed Select Turbo Frequency (SST-TF)
+# [false] Disable Intel Speed Select Base Frequency (SST-TF)
+sst_tf_configuration_enabled: false
 
 # (CentOS 7 only) install real time kernel and related packages for flexran
 install_real_time_package: false
diff --git a/sw_config/bmra/Dockerfile b/sw_config/bmra/Dockerfile
index 3f21241..d464822 100644
--- a/sw_config/bmra/Dockerfile
+++ b/sw_config/bmra/Dockerfile
@@ -10,7 +10,7 @@ RUN yum -y update && \
     yum -y install git epel-release python36 python-netaddr && \
     yum -y install python-pip && \
     pip install --no-cache-dir pip==9.0.3 && \
-    pip install --no-cache-dir ansible==2.9.17 jmespath && \
+    pip install --no-cache-dir ansible==2.9.20 jmespath && \
     pip install --no-cache-dir jinja2 --upgrade
 
 CMD ["bash"]
diff --git a/sw_config/bmra/patched_cmk_build.yml b/sw_config/bmra/patched_cmk_build.yml
index 0b5c774..a424c55 100644
--- a/sw_config/bmra/patched_cmk_build.yml
+++ b/sw_config/bmra/patched_cmk_build.yml
@@ -1,37 +1,12 @@
-# SPDX-FileCopyrightText: 2020 Intel Corporation.
+# SPDX-FileCopyrightText: 2021 Intel Corporation.
 #
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: install epel-release on Red Hat based OS
-  package: name=epel-release
-  when: ansible_os_family == 'RedHat'
-
-# note: on Ubuntu, pip is installed via install_dependencies
-- name: install pip
-  package:
-    name: python-pip
-  when:
-    - ansible_distribution in ["RedHat", "CentOS"]
-    - ansible_distribution_version < '8'
-
-- name: install pip
-  package:
-    name: python3-pip
-  when:
-    - ansible_distribution in ["RedHat", "CentOS"]
-    - ansible_distribution_version >= '8'
-
 - name: install dependencies
   include_role:
     name: install_dependencies
 
-- name: install Python dependencies
-  pip:
-    name:
-      - setuptools
-      - docker
-
 - name: clone CMK repository
   git:
     repo: "{{ cmk_git_url }}"
@@ -61,25 +36,42 @@
 - name: build CMK image
   make:
     chdir: "{{ cmk_dir }}"
+  when: container_runtime == "docker"
 
 # NOTE(przemeklal): this fixes problem in CMK with ImagePullPolicy hardcoded to Never and the pod is scheduled on controller node
 - name: tag CMK image
   command: docker tag cmk:{{ cmk_img_version }} {{ registry_local_address }}/cmk:{{ cmk_img_version }}
   changed_when: true
+  when: container_runtime == "docker"
 
 - name: push CMK image to local registry
   command: docker push {{ registry_local_address }}/cmk:{{ cmk_img_version }}
+  changed_when: true
   when:
+    - container_runtime == "docker"
     - inventory_hostname == groups['kube-node'][0]
+
+- name: build and tag CMK image
+  command: podman build -f Dockerfile -t {{ registry_local_address }}/cmk:{{ cmk_img_version }}
+  args:
+    chdir: "{{ cmk_dir }}"
   changed_when: true
+  when: '"docker" not in container_runtime'
+
+- name: push CMK image to local registry
+  command: podman push {{ registry_local_address }}/cmk:{{ cmk_img_version }}
+  changed_when: true
+  when:
+    - inventory_hostname == groups['kube-node'][0]
+    - '"docker" not in container_runtime'
 
-- name: clean up any preexisting certs/key/CSR files
+- name: clean up any pre-existing certs/key/CSR files
   file: path=/etc/ssl/cmk state=absent
   when: inventory_hostname == groups['kube-master'][0]
   failed_when: false
   become: yes
 
-- name: delete any preexisting certs/key/CSR from Kubernetes
+- name: delete any pre-existing certs/key/CSR from Kubernetes
   command: kubectl delete csr cmk-webhook-{{ item }}.{{ cmk_namespace }}
   when: inventory_hostname == groups['kube-master'][0]
   failed_when: false
@@ -212,7 +204,7 @@
   when:
     - inventory_hostname == groups['kube-master'][0]
 
-- name: get approved server  certificate
+- name: get approved server certificate
   shell: kubectl get csr cmk-webhook-server.{{ cmk_namespace }} -o jsonpath='{.status.certificate}'
   args:
     chdir: "/etc/ssl/cmk/"
@@ -272,8 +264,13 @@
 - name: restart kube-apiserver after updating admission control configuration
   when: inventory_hostname == groups['kube-master'][0]
   block:
-    - name: remove kube-apiserver Docker container
-      shell: docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f
+    - name: remove kube-apiserver container
+      # noqa 305 - shell is used intentionally here
+      shell: >-
+        {{ (container_runtime == 'docker') | ternary('docker ps -af name=k8s_kube-apiserver* -q |
+        xargs --no-run-if-empty docker rm -f',
+        'crictl ps -a --name=kube-apiserver* -q |
+        xargs --no-run-if-empty crictl rm -f')  }}
       args:
         executable: /bin/bash
       register: remove_apiserver_container
@@ -345,11 +342,11 @@
   when:
     - inventory_hostname == groups['kube-master'][0]
 
-# remove any preexisting configmaps before cmk redeployment
-- name: remove any preexisting configmaps before CMK deployment
+# remove any pre-existing configmaps before cmk redeployment
+- name: remove any pre-existing configmaps before CMK deployment
   command: kubectl delete cm cmk-config-{{ inventory_hostname }}
   when:
-    - inventory_hostname in cmk_hosts_list.split(',')
+    - inventory_hostname in (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else [])
   delegate_to: "{{ groups['kube-master']|first }}"
   failed_when: false
 
diff --git a/sw_config/bmra/patched_k8s.yml b/sw_config/bmra/patched_k8s.yml
index 5dfc3bd..fb0d43a 100644
--- a/sw_config/bmra/patched_k8s.yml
+++ b/sw_config/bmra/patched_k8s.yml
@@ -30,6 +30,10 @@
           {%- endif -%}
         enable_admission_plugins_prepare: >-
           [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}]
+        bmra_docker_version: >-
+          {% if ansible_distribution_version >= '21.04' %}latest{% else %}19.03{%endif %}
+        flannel_backend_type: >-
+          {% if ansible_distribution_version >= '21.04' %}host-gw{% else %}vxlan{%endif %}
         kube_config_dir: /etc/kubernetes
     - name: set kube_cert_dir
       set_fact:
@@ -38,15 +42,32 @@
   environment: "{{ proxy_env | d({}) }}"
   any_errors_fatal: true
 
+- hosts: all
+  tasks:
+    - name: add docker runtime vars
+      set_fact:
+        container_manager: docker
+        docker_iptables_enabled: true
+        docker_dns_servers_strict: false
+        docker_version: "{{ bmra_docker_version }}"
+      when: container_runtime == "docker"
+    - name: add containerd runtime vars
+      set_fact:
+        container_manager: containerd
+        etcd_deployment_type: host
+        containerd_extra_args: |2
+                  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry_local_address }}"]
+                    endpoint = ["https://{{ registry_local_address }}"]
+                  [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry_local_address }}".tls]
+                    ca_file   = "/etc/containers/certs.d/{{ registry_local_address }}/ca.crt"
+      when: container_runtime == "containerd"
 - name: run kubespray
   import_playbook: kubespray/cluster.yml
   vars:
     kubeadm_enabled: true
     multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf
-    docker_iptables_enabled: true
-    docker_dns_servers_strict: false
+    nginx_image_tag: 1.21.1
     override_system_hostname: false
-    docker_version: '19.03'
     kube_proxy_mode: iptables
     enable_nodelocaldns: false
     system_reserved: true
@@ -105,6 +126,7 @@
     - name: restart docker daemon to recreate iptables rules
       systemd: name=docker state=restarted
       become: yes
+      when: container_runtime == "docker"
     - name: restart kubelet to trigger static pods recreation
       systemd: name=kubelet state=restarted
       become: yes
@@ -144,7 +166,7 @@
   roles:
     - role: cluster_defaults
       tags: defaults
-    - role: docker_registry
+    - role: container_registry
       tags: registry
     - role: dockerhub_credentials
       when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file"
diff --git a/sw_config/bmra/patched_kubespray_requirements.txt b/sw_config/bmra/patched_kubespray_requirements.txt
index b6cf112..cceb6ff 100644
--- a/sw_config/bmra/patched_kubespray_requirements.txt
+++ b/sw_config/bmra/patched_kubespray_requirements.txt
@@ -2,10 +2,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-ansible==2.9.17
-jinja2==2.11.1
+ansible==2.9.20
+cryptography==2.8
+jinja2==2.11.3
 netaddr==0.7.19
 pbr==5.4.4
 jmespath==0.9.5
 ruamel.yaml==0.16.10
+MarkupSafe==1.1.1
 ruamel.yaml.clib==0.2.2
diff --git a/sw_config/bmra/patched_packages.yml b/sw_config/bmra/patched_packages.yml
index 90a8519..14d7291 100644
--- a/sw_config/bmra/patched_packages.yml
+++ b/sw_config/bmra/patched_packages.yml
@@ -19,7 +19,7 @@
       - setuptools<=44
     extra_args: --upgrade
 
-#pinned python package versions
+#pinned python packages versions
 - name: install Python packages
   pip:
     name:
@@ -27,7 +27,12 @@
       - ruamel.yaml.clib==0.2.2
       - ruamel.yaml==0.16.13
       - cachetools=={{ (ansible_os_family == 'RedHat' and ansible_distribution_version < '8') | ternary('3.1.1', '4.2.1') }}
+      - markupsafe==1.1.1
+      - jinja2==2.11.3
       - openshift==0.11.2
+      - six>=1.15.0
+      - websocket-client==0.58.0
+      - oauthlib==3.1.0
     state: present
   register: pip_result
   retries: 5
diff --git a/sw_config/bmra/patched_preflight.yml b/sw_config/bmra/patched_preflight.yml
new file mode 100644
index 0000000..41b7efd
--- /dev/null
+++ b/sw_config/bmra/patched_preflight.yml
@@ -0,0 +1,544 @@
+# SPDX-FileCopyrightText: 2021 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+# Preflight: ALL checks must PASS
+# Only assert issues (do NOT change anything)
+#
+# Manual run: 'ansible-playbook -i inventory.ini playbooks/preflight.yml --flush-cache'
+#
+# Summary:
+# On Ansible Host (localhost):
+# - Check Ansible version (match)
+# - Check Python version (min)
+# - Check Group Vars (exist)
+# - Check CMK Hosts (valid targets)
+# On All targets (k8s-cluster):
+# - Check Linux Distro
+# - Check Hostnames (match Inventory)
+# - Check CMK Config (isolcpus defined)
+# - Check isolcpus Total (not more than actual)
+# - Check isolcpus IDs (valid on system)
+# - Check isolcpus OS Reserved (not 0,1,etc)
+# On Worker Nodes Only (kube-node):
+# - Check DP Interfaces (is not empty)
+# - Check DP Interfaces Name (optional)
+# - Check DP Interfaces Bus Info (pciid)
+# - Check QAT Devices Bus Info (pciid)
+# - Check QAT SRIOV VFs (max)
+# - Check SGX configuration
+# - Check OVS DPDK Dependencies (for 1G Hugepages)
+# - Check VPP Dependencies (for 2M Hugepages)
+# - Check CNI Dependencies (for OVS DPDK or VPP and Hugepages)
+# - Check SST (not on RHEL 8.2 or old OSs)
+# - Warn BIOS VT-d (should be enabled)
+# - Warn BIOS Hyper-Threading (should be enabled)
+# - Warn collectd (kernel update needed on old OSs)
+# - Check OVS DPDK Version Compatability (for OVS support)
+
+# additional vars required:
+#  bmra_supported_ansible: # must be version
+#  bmra_supported_python: # min version
+#  bmra_supported_distros: [] # list
+#  bmra_supported_distros_versions: [] # list
+
+
+##################################
+# Prerequisites for Ansible Host #
+##################################
+- hosts: 127.0.0.1
+  connection: local
+  vars:
+    bmra_supported_ansible: 2.9.20
+    bmra_supported_python: 2.7
+
+  tasks:
+
+    - debug: msg="Ansible version is {{ ansible_version.string }}"
+    - name: Check Ansible Version
+      assert:
+        that: (ansible_version.full is version_compare(bmra_supported_ansible, '=='))
+        msg: "Ansible version must be {{ bmra_supported_ansible }}. Please update"
+
+    - debug: msg="Python version is {{ ansible_python_version }}"
+    - name: Check Python Version
+      assert:
+        that: (ansible_python_version is version_compare(bmra_supported_python, '>='))
+        msg: "Python version must be at least {{ bmra_supported_python }}. Please update"
+
+    - name: Read Group Vars
+      stat:
+        path: "{{ inventory_dir }}/group_vars/"
+      register: group_vars_details
+
+    - name: Check Group Vars
+      assert:
+        that: "group_vars_details.stat.exists and group_vars_details.stat.isdir"
+        msg: "File group_vars/all.yml does NOT exist. Must be created per Guide"
+
+    - debug:
+        msg:
+          - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml)
+          - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml)
+          - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml)
+          - all targets = {{ groups['all'] }} (inventory.ini)
+      when: cmk_enabled is defined  # CMK expected true for all profiles except basic
+
+    - name: Check Intel CMK Hosts
+      assert:
+        that: "item in groups['all']"
+        msg: "Hostname '{{ item }}' is NOT a valid target from inventory.  Please correct the cmk_hosts_list or disable the CMK feature in group vars"
+      with_items: "{{ (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) }}"
+      when: cmk_enabled is defined and cmk_enabled and not cmk_use_all_hosts
+
+
+##############################################
+# Prerequisites for Control and Worker Nodes #
+##############################################
+- hosts: k8s-cluster
+  vars:
+    bmra_supported_distros: [CentOS, RedHat, Ubuntu]
+    bmra_supported_distros_versions: ['7.6', '7.8', '7.9', '8.2', '8.3', '8.4', '18.04', '20.04', '21.04']
+    isolcpus_ranges: []
+    isolcpus_discretes: []
+
+  tasks:
+
+    - debug: msg="Linux distribution on target is {{ ansible_distribution }} {{ ansible_distribution_version }} {{ ansible_distribution_release }}"
+    - name: Check Linux Distro and Version
+      assert:
+        that: "ansible_distribution in bmra_supported_distros and ansible_distribution_version in bmra_supported_distros_versions"
+        msg:
+          - Linux distribution {{ ansible_distribution }} {{ ansible_distribution_version }} on target '{{ inventory_hostname }}' is NOT supported
+          - Must be one of {{ bmra_supported_distros }} and version {{ bmra_supported_distros_versions }}
+
+#    - name: Check Linux Across Cluster
+# TODO ?? Linux OS must be the same on all targets (no mix-n-match)
+
+    - name: regather network facts in case hostname recently changed
+      setup:
+        gather_subset: network
+    - debug: msg="Inventory target '{{ inventory_hostname }}' has the actual system hostname '{{ ansible_hostname }}'"
+    - name: Check Inventory Hostnames
+      debug:
+        msg:
+          - "Target '{{ inventory_hostname }}' in inventory does NOT match the actual system hostname '{{ ansible_hostname }}'."
+          - "If it's done intentionally, please ignore this message."
+      when:
+        - inventory_hostname != ansible_hostname
+
+# Early check if SELinux is configured properly
+    - block:
+      - name: "Collect packages facts"
+        package_facts:
+      - debug:
+          msg:
+            - "Current SELinux status:"
+            - "status: {{ ansible_selinux.status | default('') }}"
+            - "policy version:  {{ ansible_selinux.policyvers | default('') }}"
+            - "type: {{ ansible_selinux.type | default('') }}"
+            - "mode: {{ ansible_selinux.mode | default('') }}"
+            - "config_mode: {{ ansible_selinux.config_mode | default('') }}"
+
+      - name: check selinux condition possibly causing system boot failure
+        debug:
+          msg:
+            - "Current SELinux setup might cause the system possibly will not boot up on next reboot."
+            - "Please, check SELinux settings and set it up according to the documentation."
+        when:
+          - "'selinux-policy' not in ansible_facts.packages"
+          - "'selinux-policy-targeted' not in ansible_facts.packages"
+      when:
+        - ansible_os_family == "RedHat"
+
+# STORY: "cmk requires isolcpus to be configured"
+    - block:
+      - debug:
+          msg:
+            - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml)
+            - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml)
+            - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml)
+            - cmk_shared_num_cores = {{ cmk_shared_num_cores }} (group_vars/all.yml)
+            - cmk_exclusive_num_cores = {{ cmk_exclusive_num_cores }} (group_vars/all.yml)
+            - isolcpus_enabled = {{ isolcpus_enabled }} (host_vars)
+            - isolcpus = {{ isolcpus }} (host_vars)
+            - ansible_processor_count = {{ ansible_processor_count }}
+            - ansible_processor_cores = {{ ansible_processor_cores }}
+            - ansible_processor_threads_per_core = {{ ansible_processor_threads_per_core }}
+            - ansible_processor_vcpus = {{ ansible_processor_vcpus }}
+            - CPUs Reserved for OS = 0...{{ ansible_processor_count - 1 }}
+#            - CPUs Reserved for OS = {{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }} # [E207] Nested jinja pattern
+
+      - name: Check Intel CMK Config
+        assert:
+          that: ({{ cmk_enabled }} and {{ isolcpus_enabled }} and "{{ isolcpus }}" | length > 0)
+          msg:
+            - Incorrect configuration pertaining Intel CMK. Conflicting or improper values detected
+            - When Intel CMK is enabled, CPUs isolation ('isolcpus') must be set according to the example file for host_vars. Please correct the configuration
+
+      - name: Split isolcpus Groups
+        set_fact:
+          isolcpus_groups: "{{ isolcpus.split(',') }}"
+
+      - debug: msg="isolcpus_groups = {{ isolcpus_groups }}"
+
+      - name: Filter isolcpus Ranges
+        set_fact:
+          isolcpus_ranges: "{{ isolcpus_ranges + [item] }}"
+        with_items: "{{ isolcpus_groups }}"
+        when: ("-" in item)
+
+      - debug: msg="isolcpus_ranges = {{ isolcpus_ranges }}"
+
+      - name: Filter isolcpus Discretes
+        set_fact:
+          isolcpus_discretes: "{{ isolcpus_discretes + [item] }}"
+        with_items: "{{ isolcpus_groups }}"
+        when: ("-" not in item)
+
+      - debug: msg="isolcpus_discretes = {{ isolcpus_discretes }}"
+
+      - name: Build isolcpus List
+        set_fact:
+          isolcpus_list: "{{ isolcpus_list | default([]) | union(isolcpus_discretes) | union([item]) }}"
+        with_sequence: "{{ isolcpus_ranges }}"
+
+      - debug: msg="isolcpus_list = {{ isolcpus_list }}"
+
+      - name: Check isolcpus Total
+        assert:
+          that: "{{ isolcpus_list | length }} <= ansible_processor_vcpus"
+          msg:
+            - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected
+            - The number of isolcpus {{ isolcpus_list | length }}, exceeds total CPUs on target {{ ansible_processor_vcpus }}. Please correct the configuration
+        when: isolcpus is defined
+
+      - name: Check isolcpus IDs
+        assert:
+          that: "item | int <= ansible_processor_vcpus"
+          msg:
+            - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected
+            - The CPU ID {{ item }} set for isolcpus is NOT actually present on target. Please correct the configuration
+        with_items: "{{ isolcpus_list }}"
+        when: isolcpus is defined
+
+      - name: Check isolcpus OS Reserved
+        assert:
+          that: "item not in isolcpus_list"
+          msg:
+            - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected
+            - The CPU ID 0...{{ ansible_processor_count - 1 }} should NOT be set for isolcpus. Please correct the configuration
+        with_items: "{{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }}"
+        when: isolcpus is defined
+
+#TODO relationship between cmk shared/exclusive cores and isolcpus
+
+      when:
+        - cmk_enabled is defined
+        - (not cmk_hosts_list is defined) or (inventory_hostname in cmk_hosts_list)  #CMK expected true for all profiles except basic
+      # {% if not cmk_use_all_hosts %}
+        - "'kube-node' in group_names"
+      # {% endif %}
+
+
+####################################
+# Prerequisites for Worker Node(s) #
+####################################
+- hosts: kube-node
+  vars:
+    phy_nics_pciids: []
+
+  tasks:
+
+# STORY: "nic bus info specified is present on system"
+    - debug:
+        msg: "Dataplane (DP) interface(s) defined in host_vars = {{ dataplane_interfaces }}"
+      when: dataplane_interfaces is defined
+
+    - name: Check DP Interfaces
+      assert:
+        that: "dataplane_interfaces != []"
+        msg: "Dataplane (DP) interface(s) on target '{{ ansible_hostname }}' must be set in host_vars. Please correct the configuration"
+      when:
+        - dataplane_interfaces is defined
+        - (update_nic_drivers is defined and update_nic_drivers) or
+          (install_ddp_packages is defined and install_ddp_packages) or
+          (sriov_cni_enabled is defined and sriov_cni_enabled) or
+          (sriov_network_operator_enabled is defined and sriov_network_operator_enabled)
+
+    - debug:
+        msg: "Network interfaces present on target '{{ ansible_hostname }}' = {{ ansible_interfaces }}"
+
+    - name: Read Physical NICs PCIIDs
+      set_fact:
+        phy_nics_pciids: "{{ phy_nics_pciids + [ ansible_facts[item]['pciid'] ] }}"
+      with_items: "{{ ansible_interfaces }}"
+      when: ansible_facts[item]['pciid'] is defined and ansible_facts[item]['type'] == "ether"
+
+    - debug: msg="PCI Slots for the NICs on target '{{ ansible_hostname }}' = {{ phy_nics_pciids }}"
+
+    - name: Check DP Interfaces Names
+      assert:
+        that: ("{{ item.name }}" in {{ ansible_interfaces }})
+        msg: "Dataplane interface '{{ item.name }}' defined in host_vars does NOT exist on target. Please correct the configuration"
+      with_items: "{{ dataplane_interfaces }}"
+      when: dataplane_interfaces is defined and dataplane_interfaces != []
+      ignore_errors: True
+
+    - name: Check DP Interfaces Bus Info
+      assert:
+        that: ("{{ item.bus_info }}" in "{{ phy_nics_pciids }}")
+        msg: "Dataplane interface '{{ item.name }}' defined with PCI ID '{{ item.bus_info }}' does NOT exist on target. Please correct the configuration"
+      with_items: "{{ dataplane_interfaces }}"
+      when: dataplane_interfaces is defined and dataplane_interfaces != []
+      ignore_errors: True
+
+
+# QAT Devices list is okay to be left empty (default), but if was defined, device(s) must exist on target
+    - debug:
+        msg: "QAT device(s) defined in host_vars = {{ qat_devices }}"
+      when: qat_devices is defined
+
+    - name: Read QAT PCIIDs
+      shell: lshw -businfo -numeric | grep -i quickassist
+      register: lshw_qat
+      ignore_errors: True
+      when: qat_devices is defined
+
+    - debug:
+        msg: "QAT devices found on target = {{ lshw_qat.stdout }}"
+      when: qat_devices is defined
+
+    - name: Check QAT Devices' Bus Info
+      assert:
+        that: ("{{ item.qat_id }}" in """{{ lshw_qat.stdout }}""")
+        msg: "QAT device '{{ item.qat_dev }}' defined with PCI ID '{{ item.qat_id }}' does NOT exist on target. Please correct the configuration"
+      with_items: "{{ qat_devices }}"
+      when: qat_devices is defined and qat_devices != []
+      ignore_errors: True
+
+# STORY: "qat_sriov_numvfs should not exceed max supported (16) per each dev_ID"
+    - debug:
+        msg:
+          - qat_sriov_numvfs for {{ item.qat_id }} = {{ item.qat_sriov_numvfs }} (host_vars)
+          - update_qat_drivers = {{ update_qat_drivers }} (host_vars)
+      with_items: "{{ qat_devices }}"
+      when: qat_devices is defined and qat_devices != [] # update_qat_drivers expected as 'true' for all profiles except basic
+
+    - name: Check QAT SRIOV VFs
+      assert:
+        that: ({{ item.qat_sriov_numvfs }} <= 16)
+        msg:
+          - Incorrect configuration pertaining QAT SRIOV. Conflicting or improper values detected
+          - When SRIOV VFs are set for QAT, max value is 16 for each ID (max 48 total per card). Please correct the configuration
+      with_items: "{{ qat_devices }}"
+      when:
+        - update_qat_drivers is defined and update_qat_drivers
+        - qat_devices is defined and qat_devices != []
+# OpenSSL & OpenSSL*Engine must only be configured / installed when update_qat_drivers is set to 'true' and qat_devices is defined in host vars
+    - name: check OpenSSL and OpenSSL*Engine requirements
+      assert:
+        that:
+          - update_qat_drivers
+          - qat_devices is defined and qat_devices != []
+        fail_msg: "OpenSSL & OpenSSL*Engine will only configured if update_qat_drivers is set to 'true' & qat_devices is defined in host vars"
+        success_msg: "OpenSSL & OpenSSL*Engine verification completed"
+      when: openssl_install is defined and openssl_install
+
+    - name: check KMRA requirements
+      assert:
+        that:
+          - sgx_dp_enabled
+        fail_msg: "KMRA installation requires sgx_dp_enabled set to 'true'"
+        success_msg: "KMRA requirements verified"
+      when: kmra_enabled is defined and kmra_enabled
+
+    - name: check SGX configuration
+      assert:
+        that:
+          - sgx_enabled
+        fail_msg: "SGX drivers installation requires sgx_enabled set to 'true'"
+        success_msg: "SGX configuration verified"
+      when:
+        - sgx_dp_enabled is defined and sgx_dp_enabled
+        - (ansible_distribution == 'Ubuntu' and ansible_distribution_version != '21.04')
+          or (ansible_os_family == 'RedHat' and ansible_distribution_version != '8.4')
+
+    - name: check NFD configuration
+      assert:
+        that:
+          - nfd_enabled
+        fail_msg: "SGX DP requires nfd_enabled set to 'true'"
+        success_msg: "NFD configuration verified"
+      when: sgx_dp_enabled is defined and sgx_dp_enabled
+
+    - name: check kmra_pccs_api_key presence
+      assert:
+        that:
+          - kmra_pccs_api_key is defined
+        fail_msg:
+          - "kmra_pccs_api_key is not defined"
+        success_msg: "kmra_pccs_api_key presence is verified"
+      when:
+        - kmra_enabled is defined and kmra_enabled
+
+    - name: check PCCS API key length
+      assert:
+        that:
+          - kmra_pccs_api_key | length == 32
+        fail_msg: "PCCS API Key should be 32 bytes long"
+        success_msg: "PCCS API key length verified"
+      when:
+        - kmra_enabled is defined and kmra_enabled
+
+    - name: check PCCS API key is not a placeholder
+      assert:
+        that:
+          - kmra_pccs_api_key is defined
+          - kmra_pccs_api_key != "ffffffffffffffffffffffffffffffff"
+        fail_msg:
+          - "Please, visit https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'"
+          - "to generate PCCS API key."
+          - "PCCS API key is essential for KMRA deployment and usage."
+        success_msg: "PCCS API key verified"
+      when:
+        - kmra_enabled is defined and kmra_enabled
+
+# STORY: "vpp/ovsdpdk require hugepage enabled and configured"
+    - debug:
+        msg:
+          - vpp_enabled = {{ vpp_enabled }} (host_vars)
+          - example_net_attach_defs = {{ example_net_attach_defs }} (group_vars/all.yml)
+          - userspace_ovs_dpdk = {{ example_net_attach_defs['userspace_ovs_dpdk'] }} (group_vars/all.yml)
+          - userspace_vpp = {{ example_net_attach_defs['userspace_vpp'] }} (group_vars/all.yml)
+          - sriov_net_dp = {{ example_net_attach_defs['sriov_net_dp'] }} (group_vars/all.yml)
+          - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars)
+          - sriov_cni_enabled = {{ sriov_cni_enabled }} (host_vars)
+          - sriov_network_operator_enabled = {{ sriov_network_operator_enabled }} (host_vars)
+          - bond_cni_enabled = {{ bond_cni_enabled }} (host_vars)
+          - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars)
+          - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars)
+          - hugepages_enabled = {{ hugepages_enabled }} (host_vars)
+          - default_hugepage_size = {{ default_hugepage_size }} (host_vars)
+          - number_of_hugepages = {{ number_of_hugepages }} (host_vars)
+      when: vpp_enabled is defined #host_vars
+
+    - name: Check OVS DPDK Dependencies
+      assert:
+        that: >-
+          ({{ ovs_dpdk_enabled }} and not {{ vpp_enabled }} and {{ hugepages_enabled }} and
+          "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0)
+          or {{ vpp_enabled }}
+        msg:
+          - Incorrect configuration pertaining OVS DPDK. Conflicting or improper values detected
+          - When OVS DPDK is enabled, VPP must be disabled and Hugepages must be set to 1G according to host_vars example. Please correct the configuration
+      when: ovs_dpdk_enabled is defined and ovs_dpdk_enabled
+
+    - name: Check VPP Dependencies
+      assert:
+        that: >-
+          ({{ vpp_enabled }} and not {{ ovs_dpdk_enabled }} and {{ hugepages_enabled }} and
+          "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0)
+          or {{ ovs_dpdk_enabled }}
+        msg:
+          - Incorrect configuration pertaining VPP. Conflicting or improper values detected
+          - When VPP is enabled, OVS DPDK must be disabled and Hugepages must be set to 2M according to host_vars example. Please correct the configuration
+      when: vpp_enabled is defined and vpp_enabled
+
+
+# STORY: "cnis require net-attach-defs to be enabled"
+    - name: Check CNI Config
+      assert:
+        that: >-
+          ({{ userspace_cni_enabled }} and {{ ovs_dpdk_enabled }} and {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and not {{ vpp_enabled }} and
+          not {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and
+          "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0)
+          or ({{ userspace_cni_enabled }} and not {{ ovs_dpdk_enabled }} and not {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and {{ vpp_enabled }}
+          and {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and
+          "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0)
+        msg:
+          - Incorrect configuration pertaining CNI. Conflicting or improper values detected.
+          - When CNI is enabled, either OVS DPDK either VPP must be enabled and Hugepages must be according to example files. Please correct the configuration
+      when: userspace_cni_enabled is defined and userspace_cni_enabled
+
+
+# STORY: "If SST enabled, confirm minimum kernel or kernel_update specified"
+    - name: Check SST # see Jira NPF-1545
+      assert:
+        that: (not sst_bf_configuration_enabled)
+        msg: "SST-BF is NOT supported on {{ ansible_distribution }} {{ ansible_distribution_version }}. Please use a different OS or disable this feature"
+      when:
+        - sst_bf_configuration_enabled is defined
+        - (ansible_distribution == "RedHat" and ansible_distribution_version == '8.2') or ansible_distribution_version in ['7.6', '7.8', '7.9', '18.04']
+      ignore_errors: True
+
+# STORY: Intel VT-d should be enabled in BIOS
+    - name: Check Intel VT-d
+      shell: dmesg | grep DMAR | grep remapping
+      register: dmesg_dmar_remap
+      ignore_errors: True
+      changed_when: False
+
+    - debug: msg="dmesg >> {{ dmesg_dmar_remap.stdout }}"
+
+    - name: Warn about Intel VT-d
+      fail:
+        msg: "Warning: Intel VT-d appears DISABLED on target.  Please check BIOS under 'Advanced > Integrated IO Configuration' and Enable if necessary"
+      when: dmesg_dmar_remap.stdout|length  == 0
+      ignore_errors: True
+
+
+# STORY: CPU Hyper-Threading should be enabled in BIOS
+    - name: Warn about Hyper-Threading
+      fail:
+        msg: "Warning: CPU Hyper-Threading is DISABLED on target.  Please check BIOS under 'Advanced > Processor Configuration' and Enable if necessary"
+      when: ansible_processor_threads_per_core != 2
+      ignore_errors: True
+
+
+# STORY: "check for collectd. See Jira NPF-1687"
+    - name: Warn about collectd
+      fail:
+        msg: "Warning: On {{ ansible_distribution }} {{ ansible_distribution_version }} collectd won't work unless 'update_kernel' is enabled in group_vars"
+      when: ansible_distribution_version in ['7.6', '18.04']
+      ignore_errors: True
+
+
+# STORY: TEMPORARY: "ovs dpdk version requirements"
+    - debug:
+        msg:
+          - install_dpdk = {{ install_dpdk }} (host_vars)
+          - dpdk_version = {{ dpdk_version }} (host_vars)
+          - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars)
+          - ovs_version = {{ ovs_version }} (host_vars)
+      when:
+        - install_dpdk is defined #host_vars
+        - dpdk_version is defined #host_vars
+        - ovs_version is defined #host_vars
+        - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars
+
+    - name: Check OVS DPDK compatibility
+      assert:
+        that: >-
+          "{{ ovs_version }} == \"v2.15.0\" and {{ dpdk_version }} >= \"20.11\""
+          or "{{ ovs_version }} == \"v2.14.2\" and {{ dpdk_version }} == \"19.11.6\""
+          or "{{ ovs_version }} == \"v2.14.1\" and {{ dpdk_version }} == \"19.11.6\""
+          or "{{ ovs_version }} == \"v2.14.0\" and {{ dpdk_version }} == \"19.11.6\""
+          or "{{ ovs_version }} == \"v2.13.3\" and {{ dpdk_version }} == \"19.11.6\""
+          or "{{ ovs_version }} == \"v2.13.2\" and {{ dpdk_version }} == \"19.11.6\""
+          or "{{ ovs_version }} == \"v2.13.1\" and {{ dpdk_version }} == \"19.11.6\""
+          or "{{ ovs_version }} == \"v2.13.0\" and {{ dpdk_version }} == \"19.11.6\""
+        msg: "OVS {{ ovs_version }} does not build with DPDK version {{ dpdk_version }}. Please correct the host_vars configuration"
+      when:
+        - dpdk_version is defined #host_vars
+        - ovs_version is defined #host_vars
+        - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars
+
+
+    - meta: end_play
+
+#    - name: Print all variables/facts known for a host
+#      ansible.builtin.debug:
+#        var: hostvars[inventory_hostname]
+#        verbosity: 4
diff --git a/sw_config/bmra/patched_rhel_packages.yml b/sw_config/bmra/patched_rhel_packages.yml
index 40b22b6..687142a 100644
--- a/sw_config/bmra/patched_rhel_packages.yml
+++ b/sw_config/bmra/patched_rhel_packages.yml
@@ -10,7 +10,7 @@
     - ansible_distribution == "CentOS"
     - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3'
 
-- name: enable PowerTools repository on CentOS >= 8.3
+- name: enable powertools repository on CentOS >= 8.3
 # noqa 303 - yum is called intenionallly here
   command: yum config-manager --set-enabled powertools
   when:
@@ -88,7 +88,10 @@
   package:
     name: "{{ item }}"
     state: present
+  retries: 5
+  delay: 10
   register: source_status
+  until: source_status is not failed
   with_items:
     - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-193.el8.x86_64.rpm"
     - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-193.el8.x86_64.rpm"
@@ -103,22 +106,35 @@
     state: present
   register: source_status
   with_items:
-    - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm"
-    - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm"
-#    - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm"
-#    - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm"
+    - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm"
+    - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm"
   when:
     - ansible_distribution == "CentOS"
     - ansible_distribution_version == '8.3'
     - not update_kernel
 
+- name: pull matching kernel headers on CentOS 8.4
+  package:
+    name: "{{ item }}"
+    state: present
+  register: source_status
+  with_items:
+    - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.3.1.el8.x86_64.rpm"
+    - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.3.1.el8.x86_64.rpm"
+#    - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.el8.x86_64.rpm"
+#    - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.el8.x86_64.rpm"
+  when:
+    - ansible_distribution == "CentOS"
+    - ansible_distribution_version == '8.4'
+    - not update_kernel
+
 # pull the matching kernel headers if kernel is not updated
 - name: pull matching kernel headers from configured repos
 # noqa 503 - more than one condition, can't be a handler
   package:
     name:
-    - kernel-headers-{{ ansible_kernel }}
-    - kernel-devel-{{ ansible_kernel }}
+      - kernel-headers-{{ ansible_kernel }}
+      - kernel-devel-{{ ansible_kernel }}
   register: kernel_source
   retries: 3
   until: kernel_source is success
@@ -196,7 +212,7 @@
     - ansible_distribution in ["RedHat", "CentOS"]
     - ansible_distribution_version < '8'
 
-- name: Set python is python3
+- name: set python is python3
   alternatives:
     name: python
     path: /usr/bin/python3
@@ -214,6 +230,51 @@
     state: present
   when: ansible_distribution in ["RedHat", "CentOS"]
 
+- name: Add kubic yum repo and install updated version of podman
+  block:
+  - name: disable container-tools module
+    # noqa 305 - shell is used intentionally here
+    shell: dnf -y module disable container-tools
+  - name: enable rhcontainerbot/container-selinux repository
+    # noqa 305 - shell is used intentionally here
+    shell: dnf -y copr enable rhcontainerbot/container-selinux
+  - name: Add kubic yum repo
+    yum_repository:
+      name: devel_kubic_libcontainers_stable
+      description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
+      baseurl: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/
+      gpgcheck: yes
+      gpgkey: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key
+      keepcache: '0'
+  - name: install runc
+    dnf:
+      name: runc
+      state: present
+  - name: install podman package
+    package:
+      name: podman
+      state: present
+  when:
+    - ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2'
+    - '"docker" not in container_runtime'
+
+# SELINUX will be disabled later stage so, these packages are required when container_runtime is docker
+- name: install packages in RHEL >= 8.4 when container_runtime is docker
+  dnf:
+    name:
+      - bridge-utils
+      - lsof
+      - lvm2
+      - tcpdump
+      - iproute-tc
+      - openssh-server
+      - chrony
+      - iputils
+  when:
+    - ansible_distribution == "RedHat" and ansible_distribution_version >= '8.4'
+    - container_runtime == "docker"
+
+# Workaround - Set pip to a version that supports correct version of packages needed
 - name: use the correct pip version for CentOS 7
   pip:
     name:
@@ -221,3 +282,11 @@
   when:
     - ansible_distribution == "CentOS"
     - ansible_distribution_version < '7.9'
+
+- name: Install pip3 required by dpdk
+  yum:
+    name: python3-pip
+    state: latest
+  when:
+    - ansible_distribution == "CentOS"
+    - ansible_distribution_version < '7.9'
diff --git a/sw_config/bmra/patched_sriov_cni_install.yml b/sw_config/bmra/patched_sriov_cni_install.yml
new file mode 100644
index 0000000..ab0dc52
--- /dev/null
+++ b/sw_config/bmra/patched_sriov_cni_install.yml
@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: 2021 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+---
+- name: install dependencies
+  include_role:
+    name: install_dependencies
+
+- name: clone sriov-cni repository
+  git:
+    repo: "{{ sriov_cni_url }}"
+    dest: "{{ sriov_cni_dir }}"
+    version: "{{ sriov_cni_version }}"
+    force: yes
+
+# Ignore errors as this has been failing
+- name: build sriov-cni plugin
+  make:
+    chdir: /usr/src/sriov-cni
+  ignore_errors: true
+
+# missing go.sum entry for module providing package golang.org/x/sys/unix
+- name: Force checkout as some files change after build error
+  shell:
+    cmd: git checkout -- .
+    chdir: /usr/src/sriov-cni
+
+- name: build sriov-cni plugin
+  make:
+    chdir: /usr/src/sriov-cni
+
+- name: create /opt/cni/bin
+  file:
+   path: "/opt/cni/bin"
+   state: directory
+   recurse: yes
+   mode: 0755
+
+- name: install sriov-cni binary to /opt/cni/bin directory
+  copy:
+    src: /usr/src/sriov-cni/build/sriov
+    dest: /opt/cni/bin/sriov
+    mode: 0755
+    remote_src: yes