Update BMRA to v21.03

Update Kuberef to use most recent release of BMRA.
Also updates configuration with minor changes and additions.

Signed-off-by: Michael S. Pedersen <michaelx.pedersen@intel.com>
Change-Id: Ifc4ef114564a2085000c23347390089227c0ebd9
Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/72429
Tested-by: jenkins-ci <jenkins-opnfv-ci@opnfv.org>
Reviewed-by: Rihab Banday <rihab.banday@ericsson.com>

12 files changed, 488 insertions, 62 deletions

diff --git a/deploy.env b/deploy.env
index 85641ce..967bf90 100644
--- a/deploy.env
+++ b/deploy.env
@@ -1,6 +1,7 @@
 # Define environment variables
 
 export VENDOR=${VENDOR:-intel}
+export DISTRO=${DISTRO:-centos7}
 export INSTALLER=bmra
 export BMRA_PROFILE=$(yq r $CURRENTPATH/hw_config/$VENDOR/idf.yaml bmra.profile)
 
diff --git a/functions.sh b/functions.sh
index fce71f6..1a07ec0 100755
--- a/functions.sh
+++ b/functions.sh
@@ -210,7 +210,7 @@ provision_k8s() {
     ansible_cmd="/bin/bash -c '"
     if [[ "$DEPLOYMENT" == "k8s" ]]; then
         ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
-        ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
+        ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.17; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
     fi
     ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'"
 
@@ -226,7 +226,7 @@ if ! command -v docker; then
     done
 fi
 if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
-    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.1.0 ${PROJECT_ROOT}/container-experience-kits/
+    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 ${PROJECT_ROOT}/container-experience-kits/
     cp -r ${PROJECT_ROOT}/container-experience-kits/examples/${BMRA_PROFILE}/group_vars ${PROJECT_ROOT}/container-experience-kits/
 # NOTE The following condition/workaround will be removed once the reported issue https://github.com/intel/container-experience-kits/issues/68
 # is fixed upstream
@@ -245,6 +245,11 @@ cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk_build.yml \
     ${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/tasks/main.yml
 cp ${PROJECT_ROOT}/${INSTALLER}/patched_vfio.yml \
     ${PROJECT_ROOT}/container-experience-kits/roles/sriov_nic_init/tasks/bind_vf_driver.yml
+cp ${PROJECT_ROOT}/${INSTALLER}/ansible.cfg \
+    ${PROJECT_ROOT}/container-experience-kits/ansible.cfg
+cp ${PROJECT_ROOT}/${INSTALLER}/patched_rhel_packages.yml \
+    ${PROJECT_ROOT}/container-experience-kits/roles/bootstrap/install_packages/tasks/rhel.yml
+
 sudo docker run --rm \
 -e ANSIBLE_CONFIG=/bmra/ansible.cfg \
 -e PROFILE=${BMRA_PROFILE} \
diff --git a/hw_config/equinix-metal/idf.yaml b/hw_config/equinix-metal/idf.yaml
index d085c7c..7b6c07e 100644
--- a/hw_config/equinix-metal/idf.yaml
+++ b/hw_config/equinix-metal/idf.yaml
@@ -44,14 +44,21 @@ bmra:
   network_roles:
     sriov:
       - name: eno2
-        bus_info: "19:00.1"
-        device_info: "8086:1572:0200"
-        driver: iavf
+        pci: "19:00.1"
+        pf_driver: i40e
+        vf_driver: iavf
     sriov_dpdk:
       - name: eno4
-        bus_info: "19:00.3"
-        device_info: "8086:1572:0200"
-        driver: vfio-pci
+        pci: "19:00.3"
+        pf_driver: i40e
+        vf_driver: vfio-pci
+  device_roles:
+#    qat:
+#      - name: crypto01
+#        pci: "0000:ab:00.0"
+#        mod_type: qat_c62x
+#        pci_type: c6xx
+#        vfs: 4
   features:
     sriov:
       enable: true
@@ -66,7 +73,8 @@ bmra:
       hugepages_2M: 10240
     isolcpus:
       enable: true
-      cpus: "8-27,36-55"
+      autogenerate: true
+      cpus: "8-27,36-55" # Not used when autogenerate is true
     nfd: true
     cmk:
       enable: true
@@ -78,4 +86,8 @@ bmra:
     tas:
       enable: true
       demo_policy: false
+    bond_cni: true
     psp: true
+    qat:
+      enable: false
+      update_drivers: false
diff --git a/hw_config/ericsson-pod1/idf.yaml b/hw_config/ericsson-pod1/idf.yaml
index a160e0a..10e2433 100644
--- a/hw_config/ericsson-pod1/idf.yaml
+++ b/hw_config/ericsson-pod1/idf.yaml
@@ -73,14 +73,21 @@ bmra:
   network_roles:
     sriov:
       - name: eth2
-        bus_info: "05:00.0"
-        device_info: "8086:10f8:0200"
-        driver: vfio-pci
+        pci: "05:00.0"
+        pf_driver: i40e
+        vf_driver: vfio-pci
     sriov_dpdk:
       - name: eth3
-        bus_info: "05:00.1"
-        device_info: "8086:10f8:0200"
-        driver: vfio-pci
+        pci: "05:00.1"
+        pf_driver: i40e
+        vf_driver: vfio-pci
+  device_roles:
+#    qat:
+#      - name: crypto01
+#        pci: "0000:ab:00.0"
+#        mod_type: qat_c62x
+#        pci_type: c6xx
+#        vfs: 4
   features:
     sriov:
       enable: false
@@ -95,7 +102,8 @@ bmra:
       hugepages_2M: 10240
     isolcpus:
       enable: true
-      cpus: "4-9,14-19,24-29,34-39"
+      autogenerate: true
+      cpus: "4-9,14-19,24-29,34-39" # Not used when autogenerate is true
     nfd: true
     cmk:
       enable: true
@@ -107,4 +115,8 @@ bmra:
     tas:
       enable: true
       demo_policy: false
+    bond_cni: true
     psp: true
+    qat:
+      enable: false
+      update_drivers: false
diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml
index 07c45cc..4ce7c47 100644
--- a/hw_config/intel/idf.yaml
+++ b/hw_config/intel/idf.yaml
@@ -73,14 +73,21 @@ bmra:
   network_roles:
     sriov:
       - name: eth0
-        bus_info: "18:00.0"
-        device_info: "8086:158b:0200"
-        driver: vfio-pci
+        pci: "18:00.0"
+        pf_driver: i40e
+        vf_driver: vfio-pci
     sriov_dpdk:
       - name: eth1
-        bus_info: "18:00.1"
-        device_info: "8086:158b:0200"
-        driver: iavf
+        pci: "18:00.1"
+        pf_driver: i40e
+        vf_driver: iavf
+  device_roles:
+#    qat:
+#      - name: crypto01
+#        pci: "0000:ab:00.0"
+#        mod_type: qat_c62x
+#        pci_type: c6xx
+#        vfs: 4
   features:
     sriov:
       enable: true
@@ -95,7 +102,8 @@ bmra:
       hugepages_2M: 10240
     isolcpus:
       enable: true
-      cpus: "4-19,24-39,44-59,64-79"
+      autogenerate: true
+      cpus: "4-19,24-39,44-59,64-79" # Not used when autogenerate is true
     nfd: true
     cmk:
       enable: true
@@ -107,4 +115,8 @@ bmra:
     tas:
       enable: true
       demo_policy: false
+    bond_cni: true
     psp: true
+    qat:
+      enable: false
+      update_drivers: false
diff --git a/inventory/group_vars/all/global.yaml b/inventory/group_vars/all/global.yaml
index c88d794..8b2adf9 100644
--- a/inventory/group_vars/all/global.yaml
+++ b/inventory/group_vars/all/global.yaml
@@ -25,3 +25,6 @@ pub_key: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa.pub"
 
 # deployment type for kuberef
 deployment_type: "{{ lookup('env', 'DEPLOYMENT') }}"
+
+# OS distro from DISTRO from environment variables
+os_distro: "{{ lookup('env', 'DISTRO') }}"
diff --git a/playbooks/roles/bmra-config/templates/all.j2 b/playbooks/roles/bmra-config/templates/all.j2
index 6dc074e..1dbabe2 100644
--- a/playbooks/roles/bmra-config/templates/all.j2
+++ b/playbooks/roles/bmra-config/templates/all.j2
@@ -1,5 +1,5 @@
 ##
-##   Copyright (c) 2020 Intel Corporation.
+##   Copyright (c) 2020-2021 Intel Corporation.
 ##
 ##   Licensed under the Apache License, Version 2.0 (the "License");
 ##   you may not use this file except in compliance with the License.
@@ -18,14 +18,15 @@
 
 # Kubernetes version
 kubernetes: true
-kube_version: v1.18.8
-#kube_version: v1.17.11
-#kube_version: v1.16.14
+#kube_version: v1.20.4
+kube_version: v1.19.8
+#kube_version: v1.18.16
 
 # Run system-wide package update (apt dist-upgrade, yum update, ...)
 # Note: enabling this may lead to unexpected results
 # Tip: you can set this per host using host_vars
 update_all_packages: false
+update_kernel: true
 
 # Node Feature Discovery
 nfd_enabled: {{ bmra.features.nfd }}
@@ -33,7 +34,7 @@ nfd_build_image_locally: false
 nfd_namespace: kube-system
 nfd_sleep_interval: 60s
 
-# Intel CPU Manager for Kubernetes
+# Intel CPU Manager for Kubernetes (CMK)
 cmk_enabled: {{ bmra.features.cmk.enable }}
 cmk_namespace: kube-system
 cmk_use_all_hosts: false # 'true' will deploy CMK on the controller nodes too
@@ -42,19 +43,13 @@ cmk_shared_num_cores: {{ bmra.features.cmk.num_shared_cores }} # number of CPU c
 cmk_exclusive_num_cores: {{ bmra.features.cmk.num_exclusive_cores }} # number of CPU cores to be assigned to the "exclusive" pool on each of the nodes
 # cmk_shared_mode: packed # choose between: packed, spread, default: packed
 # cmk_exclusive_mode: packed # choose between: packed, spread, default: packed
+autogenerate_isolcpus: {{ bmra.features.isolcpus.autogenerate }}
 
 # Native CPU Manager (Kubernetes built-in)
-# Note: Enabling CMK and built-in CPU Manager is not recommended.
+# Note: Enabling CMK and built-in Native CPU Manager is NOT recommended.
 # Setting this option as "true" enables the "static" policy, otherwise the default "none" policy is used.
+# The reserved CPU cores settings are individual per each worker node, and therefore are available to configure in the host_vars file
 native_cpu_manager_enabled: false
-# Amount of CPU cores that will be reserved for the housekeeping (2000m = 2000 millicores = 2 cores)
-native_cpu_manager_system_reserved_cpus: 2000m
-# Amount of CPU cores that will be reserved for Kubelet
-native_cpu_manager_kube_reserved_cpus: 1000m
-# Explicit list of the CPUs reserved from pods scheduling.
-# Note: Supported only with kube_version 1.17 and newer, overrides native_cpu_manager_system_reserved_cpus and native_cpu_manager_kube_reserved_cpus.
-#native_cpu_manager_reserved_cpus: "0,1,2"
-# Note: All reamining unreserved CPU cores will be consumed by the workloads.
 
 # Enable Kubernetes built-in Topology Manager
 topology_manager_enabled: {{ bmra.features.topology_manager.enable }}
@@ -93,17 +88,37 @@ sriovdp_config_data: |
                     "devices": ["1889"],
                     "drivers": ["vfio-pci"]
                 }
+            },
+            {
+                "resourceName": "intel_fpga",
+                "deviceType": "accelerator",
+                "selectors": {
+                    "vendors": ["8086"],
+                    "devices": ["0d90"]
+                }
             }
         ]
     }
 
 # Intel QAT Device Plugin for Kubernetes
-qat_dp_enabled: false
+qat_dp_enabled: {{ bmra.features.qat.enable }}
 qat_dp_namespace: kube-system
+qat_dp_build_image_locally: true
 
 # Intel GPU Device Plugin for Kubernetes
 gpu_dp_enabled: false
 gpu_dp_namespace: kube-system
+gpu_dp_build_image_locally: true
+
+# Intel SGX Device Plugin for Kubernetes
+sgx_dp_enabled: false
+sgx_dp_build_image_locally: true
+# ProvisionLimit is a number of containers that can share
+# the same SGX provision device.
+sgx_dp_provision_limit: 20
+# EnclaveLimit is a number of containers that can share the
+# same SGX enclave device.
+sgx_dp_enclave_limit: 20
 
 # Intel Telemetry Aware Scheduling
 tas_enabled: {{ bmra.features.tas.enable }}
@@ -120,7 +135,7 @@ example_net_attach_defs:
 ## Proxy configuration ##
 #http_proxy: "http://proxy.example.com:1080"
 #https_proxy: "http://proxy.example.com:1080"
-#additional_no_proxy: ".example.com"
+#additional_no_proxy: ".example.com,mirror_ip"
 
 # (Ubuntu only) disables DNS stub listener which may cause issues on Ubuntu
 dns_disable_stub_listener: false
@@ -138,9 +153,16 @@ kube_pods_subnet: 10.244.0.0/16
 kube_service_addresses: 10.233.0.0/18
 kube_proxy_mode: iptables
 
+# comment this line out if you want to expose k8s services of type nodePort externally.
+kube_proxy_nodeport_addresses_cidr: 127.0.0.0/8
+
 # please leave it set to "true", otherwise Intel BMRA features deployed as Helm charts won't be installed
 helm_enabled: true
 
+# local Docker Hub mirror, if it exists
+#docker_registry_mirrors:
+#  - http://mirror_ip:mirror_port
+
 # Docker registry running on the cluster allows us to store images not avaialble on Docker Hub, e.g. CMK
 registry_local_address: "localhost:30500"
 
diff --git a/playbooks/roles/bmra-config/templates/inventory.j2 b/playbooks/roles/bmra-config/templates/inventory.j2
index 6008179..7f6cde0 100644
--- a/playbooks/roles/bmra-config/templates/inventory.j2
+++ b/playbooks/roles/bmra-config/templates/inventory.j2
@@ -2,7 +2,7 @@
 {% for node in nodes %}
 {{ idf.kubespray.hostnames[node.name] }}   ansible_host={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }} ip={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }}
 {% endfor %}
-localhost ansible_python_interpreter=/usr/bin/python2
+localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python2
 
 {% for role in ['kube-master', 'etcd', 'kube-node'] %}
 [{{ role }}]
@@ -19,7 +19,7 @@ kube-node
 
 [calico-rr]
 
-{% if deployment_type == 'k8s' %}
+{% if os_distro|lower != 'centos7' %}
 [all:vars]
 ansible_python_interpreter=/usr/bin/python3
 {% endif %}
diff --git a/playbooks/roles/bmra-config/templates/kube-node.j2 b/playbooks/roles/bmra-config/templates/kube-node.j2
index f32fbdb..09a6958 100644
--- a/playbooks/roles/bmra-config/templates/kube-node.j2
+++ b/playbooks/roles/bmra-config/templates/kube-node.j2
@@ -1,5 +1,5 @@
 ##
-##   Copyright (c) 2020 Intel Corporation.
+##   Copyright (c) 2020-2021 Intel Corporation.
 ##
 ##   Licensed under the Apache License, Version 2.0 (the "License");
 ##   you may not use this file except in compliance with the License.
@@ -20,33 +20,44 @@
 iommu_enabled: {{ bmra.features.sriov.enable }}
 
 # dataplane interface configuration list
+{% if bmra.network_roles.sriov is defined or bmra.network_roles.sriov_dpdk is defined %}
 dataplane_interfaces:
 {% for intf in bmra.network_roles.sriov %}
   - name: {{ intf.name }}
-    bus_info: "{{ intf.bus_info }}"
-    device_info: "{{ intf.device_info }}"
+    bus_info: "{{ intf.pci }}"
+    pf_driver: {{ intf.pf_driver }}
+    default_vf_driver: "{{ intf.vf_driver }}"
     sriov_numvfs: {{ bmra.features.sriov.sriov_vfs_per_port }}
-    vf_driver: {{ intf.driver }}
 {% endfor %}
 {% for intf in bmra.network_roles.sriov_dpdk %}
   - name: {{ intf.name }}
-    bus_info: "{{ intf.bus_info }}"
-    device_info: "{{ intf.device_info }}"
+    bus_info: "{{ intf.pci }}"
+    pf_driver: {{ intf.pf_driver }}
+    default_vf_driver: "{{ intf.vf_driver }}"
     sriov_numvfs: {{ bmra.features.sriov.sriov_dpdk_vfs_per_port }}
-    vf_driver: {{ intf.driver }}
 {% endfor %}
+{% else %}
+dataplane_interface: []
+{% endif %}
 
 sriov_cni_enabled: {{ bmra.features.sriov_cni }}
 
 # Bond CNI
-bond_cni_enabled: false
+bond_cni_enabled: {{ bmra.features.bond_cni }}
 
 # Install DPDK (required for SR-IOV networking)
 install_dpdk: true
 
+# DPDK version
+dpdk_version: "19.11.6"
+
+# Custom DPDK patches local path
+# dpdk_local_patches_dir: "/tmp/patches/dpdk-19.11.6"
+
 # Userspace networking
 userspace_cni_enabled: false
 ovs_dpdk_enabled: false # Should be enabled with Userspace CNI, when VPP is set to "false"; 1G hugepages required
+ovs_version: "v2.13.0"
 # CPU mask for OVS-DPDK PMD threads
 ovs_dpdk_lcore_mask: 0x1
 # Huge memory pages allocated by OVS-DPDK per NUMA node in megabytes
@@ -69,9 +80,41 @@ firmware_update_nics: []
 
 # install Intel x700 & x800 series NICs DDP packages
 install_ddp_packages: false
-
-# set how many VFs per single QAT device PF should be created
-qat_sriov_numvfs: 16
+# set 'true' to enable custom ddp package to be loaded after reboot
+enable_ice_systemd_service: false
+# Enabling this feature will install QAT drivers + services
+update_qat_drivers: {{ bmra.features.qat.update_drivers }}
+
+# qat interface configuration list
+{% if bmra.device_roles.qat is defined %}
+qat_devices:
+{% for dev in bmra.device_roles.qat %}
+  - qat_dev: {{ dev.name }}
+    qat_id: "{{ dev.pci }}"
+    qat_module_type: {{ dev.mod_type }}
+    qat_pci_type: {{ dev.pci_type }}
+    qat_sriov_numvfs: {{ dev.vfs }}
+{% endfor %}
+{% else %}
+qat_devices: []
+{% endif %}
+#  - qat_dev: crypto01                # Device name as separate QAT Symmetric Crypto devices on which qat_sriov_numvfs will be distributed
+#    qat_id: "0000:ab:00.0"           # Device QAT id one using DPDK compatible driver for VF devices to be used by vfio-pci kernel driver, replace as required
+#    qat_module_type: qat_c62x        # QAT Crypto Poll Mode Kernel Module supported are qat_dh895xcc,qat_c62x,qat_c3xxx,qat_200xx,qat_c4xxx,qat_d15xx
+#    qat_pci_type: c6xx               # QAT Crypto Poll Mode Pci Driver id supported are dh895xcc,c6xx,c3xxx,d15xx,200xx & c4xxx
+#    qat_sriov_numvfs: 12             # set how many VFs per qat_id to be created such as c6xxvf support 32 so per device will be 10+10+12=32, replace as required
+                                     # Note: If don't want to assign VFs to id leave it as 0 such as qat_sriov_numvfs: 0
+#  - qat_dev: crypto02
+#    qat_id: "0000:xy:00.0"
+#    qat_module_type: qat_c62x
+#    qat_pci_type: c6xx
+#    qat_sriov_numvfs: 10
+
+#  - qat_dev: crypto03
+#    qat_id: "0000:yz:00.0"
+#    qat_module_type: qat_c62x
+#    qat_pci_type: c6xx
+#    qat_sriov_numvfs: 10
 
 # Enables hugepages support
 hugepages_enabled: {{ bmra.features.hugepages.enable }}
@@ -85,10 +128,19 @@ hugepages_2M: {{ bmra.features.hugepages.hugepages_2M }}
 
 # CPU isolation from Linux scheduler
 isolcpus_enabled: {{ bmra.features.isolcpus.enable }}
-isolcpus: "{{ bmra.features.isolcpus.cpus }}" # Update to match group_vars requested exclusive/shared cores
+# Disable CMKs autogenerate_isolcpus in group_vars to set custom isolcpus range; Otherwise this range will be automatically generated
+# If defining range and using CMK you must account group_vars requested exclusive/shared cores
+isolcpus: "{{ bmra.features.isolcpus.cpus }}"
 
-# Max number of processors to support (physical & logical cores)
-cpu_count: 144
+# Native CPU Manager (Kubernetes built-in)
+
+native_cpu_manager_system_reserved_cpus: 2000m
+# Amount of CPU cores that will be reserved for Kubelet
+native_cpu_manager_kube_reserved_cpus: 1000m
+# Explicit list of the CPUs reserved from pods scheduling.
+# Note: Supported only with kube_version 1.17 and newer, overrides native_cpu_manager_system_reserved_cpus and native_cpu_manager_kube_reserved_cpus.
+#native_cpu_manager_reserved_cpus: "0,1,2"
+# Note: All reamining unreserved CPU cores will be consumed by the workloads.
 
 # Enable/Disable Intel PState scaling driver
 intel_pstate_enabled: true
@@ -109,9 +161,51 @@ sst_bf_configuration_enabled: false
 # [r] Revert cores to min/Turbo (set min/max to 800/3900)
 clx_sst_bf_mode: s
 
+# Intel Speed Select Base-Frequency configuration for Ice Lake (ICX) Platforms.
+# [true] Enable Intel Speed Select Base Frequency (SST-BF)
+# [false] Disable Intel Speed Select Base Frequency (SST-BF)
+# Requires `sst_bf_configuration_enabled` variable to be 'true'
+icx_sst_bf_enabled: false
+# Prioritze (SST-CP) power flow to high frequency cores in case of CPU power constraints.
+icx_sst_bf_with_core_priority: false
+
+# SST CP config
+# Variables are only examples.
+# For more information, please visit:
+# https://www.kernel.org/doc/html/latest/admin-guide/pm/intel-speed-select.html#enable-clos-based-prioritization
+# Enabling this configuration overrides `icx_sst_bf_with_core_priority`.
+sst_cp_configuration_enabled: false
+sst_cp_priority_type: 0 # 0 - Proportional, 1 - Ordered
+sst_cp_clos_groups: # configure up to 4 CLOS groups
+  - id: 0
+    frequency_weight: 0 # used only with Proportional type
+    min_MHz: 0
+    max_MHz: 25500
+  - id: 1
+    frequency_weight: 0 # used only with Proportional type
+    min_MHz: 0
+    max_MHz: 25500
+  - id: 2
+    frequency_weight: 0 # used only with Proportional type
+    min_MHz: 0
+    max_MHz: 25500
+  - id: 3
+    frequency_weight: 0 # used only with Proportional type
+    min_MHz: 0
+    max_MHz: 25500
+sst_cp_cpu_clos:
+  - clos: 0
+    cpus: 1,2,4..6,8-10
+  - clos: 1
+    cpus: 3,7
+
+
 # (CentOS 7 only) install real time kernel and related packages for flexran
 install_real_time_package: false
 
+# Intel Software Guard Extensions (SGX)
+sgx_enabled: false
+
 # Telemetry configuration
 # intel_pmu plugin collects information provided by Linux perf interface.
 enable_intel_pmu_plugin: false
diff --git a/sw_config/bmra/ansible.cfg b/sw_config/bmra/ansible.cfg
new file mode 100644
index 0000000..1808b58
--- /dev/null
+++ b/sw_config/bmra/ansible.cfg
@@ -0,0 +1,15 @@
+[ssh_connection]
+pipelining=True
+ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 -o ServerAliveCountMax=10
+
+[defaults]
+force_valid_group_names = ignore
+display_skipped_hosts = no
+host_key_checking = False
+gathering = smart
+
+fact_caching = jsonfile
+fact_caching_connection = /tmp
+fact_caching_timeout = 7200
+
+action_plugins=./action_plugins:~/.ansible/plugins/action:/usr/share/ansible/plugins/action
diff --git a/sw_config/bmra/patched_rhel_packages.yml b/sw_config/bmra/patched_rhel_packages.yml
new file mode 100644
index 0000000..7e65484
--- /dev/null
+++ b/sw_config/bmra/patched_rhel_packages.yml
@@ -0,0 +1,226 @@
+##
+##   Copyright (c) 2020-2021 Intel Corporation.
+##
+##   Licensed under the Apache License, Version 2.0 (the "License");
+##   you may not use this file except in compliance with the License.
+##   You may obtain a copy of the License at
+##
+##       http://www.apache.org/licenses/LICENSE-2.0
+##
+##   Unless required by applicable law or agreed to in writing, software
+##   distributed under the License is distributed on an "AS IS" BASIS,
+##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+##   See the License for the specific language governing permissions and
+##   limitations under the License.
+##
+---
+- name: enable PowerTools repository on CentOS >= 8 and < 8.3
+# noqa 303 - yum is called intenionallly here
+  command: yum config-manager --set-enabled PowerTools
+  when:
+    - ansible_distribution == "CentOS"
+    - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3'
+
+- name: enable PowerTools repository on CentOS >= 8.3
+# noqa 303 - yum is called intenionallly here
+  command: yum config-manager --set-enabled powertools
+  when:
+    - ansible_distribution == "CentOS"
+    - ansible_distribution_version >= '8.3'
+
+- name: enable CodeReady Linux Builder repository on RHEL 8
+  rhsm_repository:
+    name: codeready-builder-for-rhel-8-x86_64-rpms
+  when:
+    - ansible_distribution == "RedHat"
+    - ansible_distribution_version >= '8'
+
+- name: install epel-release on CentOS
+  package:
+    name: epel-release
+  when:
+    - ansible_distribution == "CentOS"
+
+- name: obtain EPEL GPG key on RHEL8
+  rpm_key:
+    state: present
+    key: https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8
+  when:
+    - ansible_distribution == "RedHat"
+    - ansible_distribution_version >= '8'
+
+- name: install epel-release on RHEL8
+  package:
+    name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+  when:
+    - ansible_distribution == "RedHat"
+    - ansible_distribution_version >= '8'
+
+- name: get full distribution versions
+  command: cat /etc/redhat-release
+  register: release
+  changed_when: true
+
+- name: set full distribution version
+  set_fact:
+    full_dist_version: "{{ release.stdout | regex_replace('.*(\\d+.\\d+.\\d\\d\\d\\d).*', '\\1') }}"
+
+- name: update CentOS Vault yum repository on CentOS 7
+  yum_repository:
+    name: C{{ full_dist_version }}-base
+    description: CentOS-{{ full_dist_version }} - Base
+    file: CentOS-Vault
+    baseurl: http://vault.centos.org/{{ full_dist_version }}/os/$basearch/
+    gpgcheck: yes
+    gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-{{ ansible_distribution_major_version }}
+    enabled: yes
+  when:
+    - ansible_distribution == "CentOS"
+    - ansible_distribution_version < '7.9'
+    - not update_kernel
+
+#- name: update CentOS Vault yum repository on CentOS 8
+#  yum_repository:
+#    name: C{{ full_dist_version }}-base
+#    description: CentOS-{{ full_dist_version }} - Base
+#    file: CentOS-Vault
+#    baseurl: http://vault.centos.org/{{ full_dist_version }}/BaseOS/$basearch/os/
+#    baseurl: http://vault.centos.org/{{ full_dist_version }}/BaseOS/Source/
+#    gpgcheck: yes
+#    gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+#    enabled: yes
+#  when:
+#    - ansible_distribution == "CentOS"
+#    - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3'
+#    - not update_kernel
+
+# CentOS-Vault repo not working for CentOS 8, so install kernel headers directly
+- name: pull matching kernel headers on CentOS 8.2
+  package:
+    name: "{{ item }}"
+    state: present
+  register: source_status
+  with_items:
+    - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-193.el8.x86_64.rpm"
+    - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-193.el8.x86_64.rpm"
+  when:
+    - ansible_distribution == "CentOS"
+    - ansible_distribution_version == '8.2'
+    - not update_kernel
+
+- name: pull matching kernel headers on CentOS 8.3
+  package:
+    name: "{{ item }}"
+    state: present
+  register: source_status
+  with_items:
+    - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm"
+    - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm"
+#    - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm"
+#    - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm"
+  when:
+    - ansible_distribution == "CentOS"
+    - ansible_distribution_version == '8.3'
+    - not update_kernel
+
+# pull the matching kernel headers if kernel is not updated
+- name: pull matching kernel headers from configured repos
+# noqa 503 - more than one condition, can't be a handler
+  package:
+    name:
+    - kernel-headers-{{ ansible_kernel }}
+    - kernel-devel-{{ ansible_kernel }}
+  register: kernel_source
+  retries: 3
+  until: kernel_source is success
+  when:
+    - not source_status.changed
+    - ansible_os_family == "RedHat"
+    - not update_kernel
+
+- name: install the 'Development tools' package group
+  package:
+    name: "@Development tools"
+  when:
+    - ansible_os_family == "RedHat"
+
+- name: install pip
+  package:
+    name: python-pip
+  when:
+    - ansible_distribution in ["RedHat", "CentOS"]
+    - ansible_distribution_version < '8'
+
+- name: install pip
+  package:
+    name: python3-pip
+  when:
+    - ansible_distribution in ["RedHat", "CentOS"]
+    - ansible_distribution_version >= '8'
+
+- name: update all packages
+  package:
+    name: '*'
+    state: latest # noqa 403
+    exclude: kernel*
+  when:
+    - ansible_os_family == "RedHat"
+    - update_all_packages | default(false)
+
+- name: update to the latest kernel and kernel headers on the Red Hat OS family
+  package:
+    name:
+    - kernel
+    - kernel-devel
+    state: latest # noqa 403
+  notify:
+    - reboot server
+  when:
+    - ansible_os_family == "RedHat"
+    - update_kernel | default(false)
+
+#note(przemeklal): fixes issue with missing selinux in packet.net CentOS 7 images
+- name: ensure selinux is installed on CentOS/RHEL 7
+  package:
+    name:
+    - policycoreutils
+    - policycoreutils-python
+    - selinux-policy
+    - selinux-policy-targeted
+    - libselinux-utils
+    - setools
+    - setools-console
+    - shtool
+    - lshw
+    state: present
+  when:
+    - ansible_distribution in ["RedHat", "CentOS"]
+    - ansible_distribution_version < '8'
+
+# Workaround for Equinix Metal CentOS 7
+- name: set selinux to permissive
+  lineinfile:
+    path: "/etc/sysconfig/selinux"
+    regexp: '^SELINUX=enforcing'
+    line: 'SELINUX=permissive'
+  when:
+    - ansible_distribution in ["RedHat", "CentOS"]
+    - ansible_distribution_version < '8'
+
+- name: Set python is python3
+  alternatives:
+    name: python
+    path: /usr/bin/python3
+    link: /usr/bin/python
+  when:
+    - ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat'
+    - ansible_distribution_version >= '8'
+
+- name: install command line tools to collect hardware details
+  package:
+    name:
+     - hwinfo
+     - inxi
+     - jq
+    state: present
+  when: ansible_distribution in ["RedHat", "CentOS"]
diff --git a/sw_config/bmra/patched_vfio.yml b/sw_config/bmra/patched_vfio.yml
index c0a6e25..81d4ab5 100644
--- a/sw_config/bmra/patched_vfio.yml
+++ b/sw_config/bmra/patched_vfio.yml
@@ -1,5 +1,5 @@
 ##
-##   Copyright (c) 2020 Intel Corporation.
+##   Copyright (c) 2020-2021 Intel Corporation.
 ##
 ##   Licensed under the Apache License, Version 2.0 (the "License");
 ##   you may not use this file except in compliance with the License.
@@ -18,17 +18,41 @@
 - name: Check that selected driver module is available
   # if modinfo fails, lookup loaded modules as modinfo might return error
   # for igb_uio and potentially other modules not included with the kernel
-  shell: "modinfo {{ vf_driver }} || grep {{ vf_driver }} /proc/modules || grep {{ vf_driver }} /lib/modules/$(uname -r)/modules.builtin"
+  shell: "modinfo {{ vf_driver.value }} || grep {{ vf_driver.value }} /proc/modules || grep {{ vf_driver.value }} /lib/modules/$(uname -r)/modules.builtin"
   register: shell_result
   ignore_errors: yes
   failed_when: no
   changed_when: no
+  with_dict: "{{ item.sriov_vfs | default({}) | combine({'default': item.default_vf_driver}) }}"
+  loop_control:
+    loop_var: vf_driver
+
+- name: pre-create empty dict for VFs
+  set_fact:
+    vfs_acc: {}
+
+- name: populate VFs dict with values
+  set_fact:
+    vfs_acc: "{{ vfs_acc | combine({idx : item.default_vf_driver}) }}"
+  loop: "{{ range(item.sriov_numvfs | default(0) | int) | list }}"
+  loop_control:
+    index_var: idx
+    loop_var: vf_default
+
+- name: update VFs dict with default drivers
+  set_fact:
+    vfs_acc: "{{ vfs_acc | combine({vf.key | regex_replace('.*_(\\d*)', '\\1') | int : vf.value}) }}"
+  loop: "{{ item.sriov_vfs | default({}) | dict2items | sort(attribute='key') }}"
+  loop_control:
+    loop_var: vf
+    extended: yes
+  when:  ansible_loop.index < (item.sriov_numvfs | default(0) | int )
 
 # get a list of VFs PCI addresses and save the configuration
 - name: attach VFs driver
   block:
     - name: fetch VFs pci addresses for a PF
-      shell: "for vf in /sys/class/net/{{ pfname }}/device/virtfn*;do basename $(readlink -f $vf);done"
+      shell: "for vf in /sys/class/net/{{ item.name }}/device/virtfn*;do basename $(readlink -f $vf);done | sort"
       register: vf_pciids
       args:
         executable: /bin/bash
@@ -37,16 +61,16 @@
     - name: save VF driver binding
       lineinfile:
         path: "{{ sriov_config_path }}/bmra_interfaces"
-        line: "{{ this_item }} {{ vf_driver }}"
-        regexp: "^{{ this_item }}"
+        line: "{{ this_item[0] }} {{ this_item[1].value }}"
+        regexp: "^{{ this_item[0] }}"
         create: yes
         owner: root
         group: root
         mode: '0600'
-      loop: "{{ vf_pciids.stdout_lines }}"
+      loop: "{{ vf_pciids.stdout_lines | zip(vfs_acc | dict2items) | list }}"
       loop_control:
         loop_var: this_item
       when:
         - vf_pciids.stderr|length == 0
         - vf_pciids.stdout_lines|length > 0
-  when: shell_result.rc == 0
+  when: shell_result.results | sum(attribute='rc') == 0