diff options
Diffstat (limited to 'sw_config/bmra')
| -rw-r--r-- | sw_config/bmra/dpdk_patch.yml | 111 | ||||
| -rw-r--r-- | sw_config/bmra/patched_cmk.yml | 84 | ||||
| -rw-r--r-- | sw_config/bmra/patched_cmk_build.yml | 376 | ||||
| -rw-r--r-- | sw_config/bmra/patched_tas.yml | 179 |
4 files changed, 750 insertions, 0 deletions
diff --git a/sw_config/bmra/dpdk_patch.yml b/sw_config/bmra/dpdk_patch.yml new file mode 100644 index 0000000..ec77b39 --- /dev/null +++ b/sw_config/bmra/dpdk_patch.yml @@ -0,0 +1,111 @@ +## +## Copyright (c) 2020 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: install dependencies + include_role: + name: install_dependencies + +- name: download DPDK + unarchive: + src: "{{ dpdk_url }}" + dest: "/usr/src" + remote_src: yes + list_files: yes + mode: 0755 + register: dpdk_download + +- name: set local dpdk directory path + set_fact: + dpdk_dir: "{{ dpdk_download.dest }}/{{ dpdk_download.files[0] }}" + +- name: run make config + make: + chdir: "{{ dpdk_dir }}" + target: config + params: + T: "{{ dpdk_target }}" + +- name: update ansible_kernel fact + setup: + filter: 'ansible_kernel' + +- name: patch DPDK (kni) + lineinfile: + path: "{{ dpdk_dir }}/kernel/linux/kni/kni_net.c" + regexp: '^\s*kni_net_tx_timeout' + line: 'kni_net_tx_timeout(struct net_device *dev, unsigned int txqueue)' + when: + - ansible_distribution == "CentOS" + - ansible_distribution_major_version == '8' + - ansible_kernel is version('4.18.0-240','>=') + +- name: enable virtio-user support + lineinfile: + path: "{{ dpdk_dir }}/build/.config" + regexp: '^CONFIG_RTE_VIRTIO_USER' + line: 'CONFIG_RTE_VIRTIO_USER=y' + mode: 0600 + +- name: enable PCAP PMD support + lineinfile: + path: "{{ dpdk_dir }}/build/.config" + regexp: '^CONFIG_RTE_LIBRTE_PMD_PCAP' + line: 'CONFIG_RTE_LIBRTE_PMD_PCAP=y' + mode: 0600 + +- name: build DPDK + make: + target: install + chdir: "{{ dpdk_dir }}" + params: + T: "{{ dpdk_target }}" + DESTDIR: install + prefix: "/usr" + +- name: find dpdk tools + find: + path: "{{ dpdk_dir }}" + patterns: "dpdk-devbind.py" + recurse: yes + register: dpdk_tools_dir + +- name: set path to dpdk usertools directory + set_fact: + dpdk_tools: "{{ dpdk_tools_dir.files[0].path }}" + +- name: load userspace modules + modprobe: + name: "{{ item }}" + state: present + with_items: + - vfio-pci + - uio + +- name: install dpdk-devbind.py in /usr/local/bin + copy: + remote_src: yes + src: "{{ dpdk_tools }}" + dest: "/usr/local/bin/dpdk-devbind.py" + mode: 0700 + owner: root + group: root + become: yes + +- name: load intel module + command: "insmod {{ dpdk_dir }}/{{ dpdk_target }}/kmod/igb_uio.ko" + register: result + failed_when: "'No such file or directory' in result.stderr" + changed_when: "'already bound' not in result.stderr" diff --git a/sw_config/bmra/patched_cmk.yml b/sw_config/bmra/patched_cmk.yml new file mode 100644 index 0000000..e97083a --- /dev/null +++ b/sw_config/bmra/patched_cmk.yml @@ -0,0 +1,84 @@ +{{- $fullName := include "cmk.fullname" . -}} +{{- $exclusiveMode := .Values.exclusiveMode -}} +{{- $numExclusiveCores := .Values.numExclusiveCores -}} +{{- $sharedMode := .Values.sharedMode -}} +{{- $numSharedCores := .Values.numSharedCores -}} +{{- $pullPolicy := .Values.image.pullPolicy -}} +{{- $image := .Values.image.repository -}} +{{- $tag := .Values.image.tag -}} +{{- $noTaint := .Values.noTaint }} +{{ range splitList "," .Values.hosts.list }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + helm.sh/hook: pre-install,pre-upgrade,pre-rollback + helm.sh/hook-weight: "10" + labels: + app: {{ $fullName }}-init-discover-{{ . }} + name: {{ $fullName }}-init-discover-{{ . }} +spec: + template: + spec: + serviceAccountName: {{ $fullName }} + restartPolicy: Never + tolerations: + - key: cmk + operator: Exists + containers: + - name: install + image: {{ $image }}:{{ $tag }} + imagePullPolicy: {{ $pullPolicy }} + command: ["/bin/bash", "-c"] + args: + - "/cmk/cmk.py install" + volumeMounts: + - mountPath: /opt/bin + name: cmk-install-dir + - name: init + image: {{ $image }}:{{ $tag }} + imagePullPolicy: {{ $pullPolicy }} + env: + - name: CMK_PROC_FS + value: "/host/proc" + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + command: ["/bin/bash", "-c"] + args: + - "/cmk/cmk.py init --exclusive-mode={{ $exclusiveMode }} --num-exclusive-cores={{ $numExclusiveCores }} --shared-mode={{ $sharedMode }} --num-shared-cores={{ $numSharedCores }}" + volumeMounts: + - mountPath: /host/proc + name: host-proc + readOnly: true + - name: discover + image: {{ $image }}:{{ $tag }} + imagePullPolicy: {{ $pullPolicy }} + env: + - name: CMK_PROC_FS + value: /host/proc + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + command: ["/bin/bash", "-c"] + args: + - "/cmk/cmk.py discover {{ if $noTaint }}--no-taint{{ end }}" + volumeMounts: + - mountPath: /host/proc + name: host-proc + readOnly: true + volumes: + - hostPath: + path: /proc + type: Directory + name: host-proc + - hostPath: + path: /opt/bin + type: DirectoryOrCreate + name: cmk-install-dir +{{ end }} diff --git a/sw_config/bmra/patched_cmk_build.yml b/sw_config/bmra/patched_cmk_build.yml new file mode 100644 index 0000000..ae0c94d --- /dev/null +++ b/sw_config/bmra/patched_cmk_build.yml @@ -0,0 +1,376 @@ +## +## Copyright (c) 2020 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: install epel-release on Red Hat based OS + package: name=epel-release + when: ansible_os_family == 'RedHat' + +# note: on Ubuntu, pip is installed via install_dependencies +- name: install pip + package: + name: python-pip + when: + - ansible_distribution in ["RedHat", "CentOS"] + - ansible_distribution_version < '8' + +- name: install pip + package: + name: python3-pip + when: + - ansible_distribution in ["RedHat", "CentOS"] + - ansible_distribution_version >= '8' + +- name: install dependencies + include_role: + name: install_dependencies + +- name: install Python dependencies + pip: + name: + - setuptools + - docker + +- name: clone CMK repository + git: + repo: "{{ cmk_git_url }}" + dest: "{{ cmk_dir }}" + version: "{{ cmk_version }}" + force: yes + +- name: patch CMK dockerfile (1/3) + lineinfile: + path: "{{ cmk_dir }}/Dockerfile" + regexp: '^FROM clearlinux' + line: 'FROM centos/python-36-centos7:latest' + +- name: patch CMK dockerfile (2/3) + lineinfile: + path: "{{ cmk_dir }}/Dockerfile" + insertafter: '^FROM centos' + line: 'USER root' + state: present + +- name: patch CMK dockerfile (3/3) + lineinfile: + path: "{{ cmk_dir }}/Dockerfile" + state: absent + regexp: '^RUN swupd' + +- name: build CMK image + make: + chdir: "{{ cmk_dir }}" + +# NOTE(przemeklal): this fixes problem in CMK with ImagePullPolicy hardcoded to Never and the pod is scheduled on controller node +- name: tag CMK image + command: docker tag cmk:{{ cmk_img_version }} {{ registry_local_address }}/cmk:{{ cmk_img_version }} + changed_when: true + +- name: push CMK image to local registry + command: docker push {{ registry_local_address }}/cmk:{{ cmk_img_version }} + when: + - inventory_hostname == groups['kube-node'][0] + changed_when: true + +- name: clean up any preexisting certs/key/CSR files + file: path=/etc/ssl/cmk state=absent + when: inventory_hostname == groups['kube-master'][0] + failed_when: false + become: yes + +- name: delete any preexisting certs/key/CSR from Kubernetes + command: kubectl delete csr cmk-webhook-{{ item }}.{{ cmk_namespace }} + when: inventory_hostname == groups['kube-master'][0] + failed_when: false + with_items: + - "client" + - "server" + +- name: create directory for CMK cert and key generation + become: yes + file: + path: /etc/ssl/cmk + state: directory + mode: 0700 + owner: root + group: root + +- name: populate CMK CSR template + template: + src: "webhook_{{ item }}_csr.json.j2" + dest: "/etc/ssl/cmk/cmk-webhook-{{ item }}-csr.json" + force: yes + mode: preserve + become: yes + with_items: + - "client" + - "server" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: get GOPATH + command: go env GOPATH + register: gopath + when: + - inventory_hostname == groups['kube-master'][0] + +- name: generate key and CSR + shell: "set -o pipefail \ + && {{ gopath.stdout }}/bin/cfssl genkey cmk-webhook-{{ item }}-csr.json | {{ gopath.stdout }}/bin/cfssljson -bare cmk-webhook-{{ item }}" + args: + chdir: "/etc/ssl/cmk/" + executable: /bin/bash + with_items: + - "client" + - "server" + when: + - inventory_hostname == groups['kube-master'][0] + become: yes + +- name: read generated server key + command: cat cmk-webhook-server-key.pem + args: + chdir: "/etc/ssl/cmk/" + register: server_key + when: + - inventory_hostname == groups['kube-master'][0] + +- name: read generated client key + command: cat cmk-webhook-client-key.pem + args: + chdir: "/etc/ssl/cmk/" + register: client_key + when: + - inventory_hostname == groups['kube-master'][0] + +- name: load generated server key + set_fact: + cmk_webhook_server_key: "{{ server_key.stdout | b64encode }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: load generated client key + set_fact: + cmk_webhook_client_key: "{{ client_key.stdout | b64encode }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: read generated client csr + command: cat cmk-webhook-client.csr + args: + chdir: "/etc/ssl/cmk/" + register: client_csr + when: + - inventory_hostname == groups['kube-master'][0] + +- name: load generated client csr + set_fact: + cmk_webhook_client_csr: "{{ client_csr.stdout | b64encode }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: read generated server csr + command: cat cmk-webhook-server.csr + args: + chdir: "/etc/ssl/cmk/" + register: server_csr + when: + - inventory_hostname == groups['kube-master'][0] + +- name: load generated server csr + set_fact: + cmk_webhook_server_csr: "{{ server_csr.stdout | b64encode }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: populate CMK Kubernetes CA CSR template + template: + src: "kube_{{ item }}_csr.yml.j2" + dest: "/etc/ssl/cmk/cmk-webhook-kube-{{ item }}-csr.yml" + force: yes + mode: preserve + with_items: + - "client" + - "server" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: send CSR to the Kubernetes API Server + command: kubectl apply -f /etc/ssl/cmk/cmk-webhook-kube-{{ item }}-csr.yml + with_items: + - "client" + - "server" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: approve request + command: kubectl certificate approve cmk-webhook-{{ item }}.{{ cmk_namespace }} + with_items: + - "client" + - "server" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: get approved server certificate + shell: kubectl get csr cmk-webhook-server.{{ cmk_namespace }} -o jsonpath='{.status.certificate}' + args: + chdir: "/etc/ssl/cmk/" + register: server_cert + when: + - inventory_hostname == groups['kube-master'][0] + retries: 30 + delay: 1 + until: server_cert.rc == 0 + +- name: get approved client certificate + shell: kubectl get csr cmk-webhook-client.{{ cmk_namespace }} -o jsonpath='{.status.certificate}' + args: + chdir: "/etc/ssl/cmk/" + register: client_cert + when: + - inventory_hostname == groups['kube-master'][0] + retries: 30 + delay: 1 + until: client_cert.rc == 0 + +- name: load generated server cert + set_fact: + cmk_webhook_server_cert: "{{ server_cert.stdout }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: load generated client cert + set_fact: + cmk_webhook_client_cert: "{{ client_cert.stdout }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: populate cmk-webhook.conf file + template: + src: "cmk-webhook.conf.j2" + dest: "/etc/kubernetes/admission-control/cmk-webhook.conf" + force: yes + mode: preserve + when: + - inventory_hostname == groups['kube-master'][0] + +- name: add MutatingAdmissionWebhook to AdmissionConfiguration + blockinfile: + path: /etc/kubernetes/admission-control/config.yaml + insertafter: "plugins:" + block: | + - name: MutatingAdmissionWebhook + configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: WebhookAdmissionConfiguration + kubeConfigFile: /etc/kubernetes/admission-control/cmk-webhook.conf + when: + - inventory_hostname == groups['kube-master'][0] + + +- name: restart kube-apiserver after updating admission control configuration + when: inventory_hostname == groups['kube-master'][0] + block: + - name: remove kube-apiserver Docker container + shell: docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f + args: + executable: /bin/bash + register: remove_apiserver_container + retries: 10 + until: remove_apiserver_container.rc == 0 + delay: 1 + - name: wait for kube-apiserver to be up + uri: + url: "https://127.0.0.1:6443/healthz" + client_cert: "/etc/kubernetes/ssl/ca.crt" + client_key: "/etc/kubernetes/ssl/ca.key" + validate_certs: no + register: result + until: result.status == 200 + retries: 120 + delay: 1 + +- name: create Helm charts directory if needed + file: + path: /usr/src/charts + state: directory + mode: 0755 + when: + - inventory_hostname == groups['kube-master'][0] + +- name: copy CMK Helm chart to the controller node + copy: + src: "{{ role_path }}/charts/cpu-manager-for-kubernetes" + dest: "/usr/src/charts/" + mode: 0755 + when: + - inventory_hostname == groups['kube-master'][0] + +# adds all kube-nodes to the list of CMK nodes +- name: build list of CMK hosts + set_fact: + cmk_hosts_list: "{{ groups['kube-node'] | join(',') }}" + when: + - not cmk_use_all_hosts + - (cmk_hosts_list is undefined) or (cmk_hosts_list | length == 0) + +- name: set values for CMK Helm chart values + set_fact: + cmk_image: "{{ registry_local_address }}/cmk" + cmk_tag: "{{ cmk_img_version }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: read ca cert + command: cat ca.crt + args: + chdir: "/etc/kubernetes/ssl/" + register: ca_cert + when: + - inventory_hostname == groups['kube-master'][0] + +- name: load ca cert + set_fact: + caBundle_cert: "{{ ca_cert.stdout | b64encode }}" + when: + - inventory_hostname == groups['kube-master'][0] + +- name: populate CMK Helm chart values template and push to controller node + template: + src: "helm_values.yml.j2" + dest: "/usr/src/charts/cmk-values.yml" + force: yes + mode: preserve + when: + - inventory_hostname == groups['kube-master'][0] + +# remove any preexisting configmaps before cmk redeployment +- name: remove any preexisting configmaps before CMK deployment + command: kubectl delete cm cmk-config-{{ inventory_hostname }} + when: + - inventory_hostname in cmk_hosts_list.split(',') + delegate_to: "{{ groups['kube-master']|first }}" + failed_when: false + +- name: install CMK helm chart + command: helm upgrade --install cmk --namespace {{ cmk_namespace }} -f /usr/src/charts/cmk-values.yml /usr/src/charts/cpu-manager-for-kubernetes + when: + - inventory_hostname == groups['kube-master'][0] + +- name: clean up any certs/key/CSR files + file: path=/etc/ssl/cmk state=absent + when: inventory_hostname == groups['kube-master'][0] + failed_when: false + become: yes diff --git a/sw_config/bmra/patched_tas.yml b/sw_config/bmra/patched_tas.yml new file mode 100644 index 0000000..633f5d7 --- /dev/null +++ b/sw_config/bmra/patched_tas.yml @@ -0,0 +1,179 @@ +## +## Copyright (c) 2020 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name : install dependencies + include_role: + name: install_dependencies + +- name: update to git2 on RHEL 7 based distros + include_role: + name: git2_install + when: + - ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' + - ansible_distribution_version < '8' + +- name: check if stable repo has already been added + command: helm repo list + register: helm_repo_list + failed_when: false + changed_when: false + +- name: add Helm stable repo + command: helm repo add stable https://charts.helm.sh/stable + when: '"https://charts.helm.sh/stable" not in helm_repo_list.stdout' + register: helm_add_result + changed_when: '"has been added to your repositories" in helm_add_result.stdout' + +- name: update Helm repo before installation of public charts + command: helm repo update + register: helm_update_result + changed_when: '"Successfully got an update" in helm_update_result.stdout' + + +- name: create Helm charts directory if needed + file: + path: /usr/src/charts + state: directory + mode: 0755 + +- name: generate cert and key + include_role: + name: create_signed_k8s_certs + vars: + secret_name: "{{ tas_extender_secret_name }}" + service_name: tas-telemetry-aware-scheduling + key_pair_name: tas + host_secrets_folder: "{{ tas_ssl_mount_path }}" + k8s_namespace: "{{ tas_namespace }}" + csr_cluster_name: "{{ cluster_name | default('cluster.local') }}" + when: tas_tls_enabled + +- name: clone TAS repository + git: + repo: "{{ tas_git_url }}" + version: "{{ tas_git_version }}" + dest: "{{ tas_dir }}" + force: yes + +- name: make build and make image - TAS + make: + target: "{{ item }}" + chdir: "{{ tas_dir }}" + loop: + - build + - image + +- name: tag TAS-controller and TAS-extender + # TAS Makefile always creates ":latest" version images + command: docker tag {{ item }}:latest {{ registry_local_address }}/{{ item }}:{{ tas_version }} + loop: + - tas-controller + - tas-extender + changed_when: false + +- name: push TAS-controller and TAS-extender image to local registry + command: docker push {{ registry_local_address }}/{{ item }}:{{ tas_version }} + loop: + - tas-controller + - tas-extender + changed_when: true + +- name: create descheduler directory if needed + file: + path: "{{ sigs_k8s_io_dir }}" + state: directory + mode: 0755 + +- name: clone Descheduler for Kubernetes + git: + repo: "{{ descheduler_git_url }}" + dest: "{{ descheduler_dir }}" + force: yes + version: "{{ descheduler_git_version }}" + +- name: install descheduler + make: + chdir: "{{ descheduler_dir }}" + +- name: copy Helm chart resource definition to controller node + copy: + src: "{{ role_path }}/charts/{{ item }}" + dest: "/usr/src/charts/" + mode: preserve + loop: + - telemetry-aware-scheduling + - tas-policy-crd.yml + +- name: populate tas Helm chart values template and push to controller node + template: + src: "tas-values.yml.j2" + dest: "/usr/src/charts/tas-values.yml" + force: yes + mode: preserve + +- name: create TASPolicy resource + command: kubectl apply -f tas-policy-crd.yml + args: + chdir: "/usr/src/charts" + changed_when: true + +- name: install TAS helm chart + command: helm upgrade --install --namespace {{ tas_namespace }} {{ tas_name }} -f tas-values.yml telemetry-aware-scheduling/ + args: + chdir: "/usr/src/charts" + retries: 5 + delay: 5 + register: result + until: result.rc == 0 + changed_when: true + +- name: Configure arguments from Kubernetes Scheduler file if they exist - dnsPolicy + lineinfile: + path: /etc/kubernetes/manifests/kube-scheduler.yaml + insertafter: "spec:" + line: " dnsPolicy: ClusterFirstWithHostNet" + regexp: " dnsPolicy: " + state: present + mode: 0600 + +- name: Configure arguments to our kube-scheduler manifest - configmap + lineinfile: + path: /etc/kubernetes/manifests/kube-scheduler.yaml + insertafter: " - kube-scheduler" + line: "{{ item.arg }}={{ item.value }}" + regexp: "{{ item.arg }}" + state: present + mode: 0600 + with_items: + - { arg: " - --policy-configmap", value: "{{ tas_name }}-telemetry-aware-scheduling-scheduler-extender-policy" } + - { arg: " - --policy-configmap-namespace", value: "{{ tas_namespace }}" } + +# TAS Demo Policy +- name: template TAS demo policy + template: + src: "tas-demo-policy.yml.j2" + dest: "/usr/src/charts/tas-demo-policy.yml" + force: yes + mode: preserve + when: + - tas_enable_demo_policy + +- name: create TAS demo policy resource + command: kubectl apply -f tas-demo-policy.yml + args: + chdir: "/usr/src/charts" + when: + - tas_enable_demo_policy |