diff options
| -rwxr-xr-x | functions.sh | 13 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/tasks/main.yaml | 2 | ||||
| -rw-r--r-- | sw_config/bmra/dockerhub_credentials/tasks/main.yml | 8 | ||||
| -rw-r--r-- | sw_config/bmra/dockerhub_credentials/vars/.gitignore | 0 | ||||
| -rw-r--r-- | sw_config/bmra/patched_k8s.yml | 166 |
5 files changed, 188 insertions, 1 deletions
diff --git a/functions.sh b/functions.sh index 6d8e923..ccc8bc7 100755 --- a/functions.sh +++ b/functions.sh @@ -161,6 +161,7 @@ get_vm_ip() { # Copy files needed by Infra engine & BMRA in the jumphost VM copy_files_jump() { vm_ip="$(get_vm_ip)" + docker_config="/opt/kuberef/docker_config" scp -r -o StrictHostKeyChecking=no \ "$CURRENTPATH"/{hw_config/"$VENDOR"/,sw_config/"$INSTALLER"/} \ "$USERNAME@${vm_ip}:$PROJECT_ROOT" @@ -169,6 +170,10 @@ copy_files_jump() { ~/.ssh/id_rsa \ "$USERNAME@${vm_ip}:.ssh/id_rsa" fi + if [ -f "$docker_config" ]; then + scp -r -o StrictHostKeyChecking=no \ + "$docker_config" "$USERNAME@${vm_ip}:$PROJECT_ROOT" + fi } # Host Provisioning @@ -229,6 +234,14 @@ if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 ${PROJECT_ROOT}/container-experience-kits/ cp -r ${PROJECT_ROOT}/container-experience-kits/examples/${BMRA_PROFILE}/group_vars ${PROJECT_ROOT}/container-experience-kits/ fi +if [ -f "${PROJECT_ROOT}/docker_config" ]; then + cp ${PROJECT_ROOT}/docker_config \ + ${PROJECT_ROOT}/${INSTALLER}/dockerhub_credentials/vars/main.yml + cp -r ${PROJECT_ROOT}/${INSTALLER}/dockerhub_credentials \ + ${PROJECT_ROOT}/container-experience-kits/roles/ + cp ${PROJECT_ROOT}/${INSTALLER}/patched_k8s.yml \ + ${PROJECT_ROOT}/container-experience-kits/playbooks/k8s/k8s.yml +fi cp ${PROJECT_ROOT}/${INSTALLER}/{inventory.ini,ansible.cfg} \ ${PROJECT_ROOT}/container-experience-kits/ cp ${PROJECT_ROOT}/${INSTALLER}/{all.yml,kube-node.yml} \ diff --git a/playbooks/roles/jump-vm/tasks/main.yaml b/playbooks/roles/jump-vm/tasks/main.yaml index 9c556da..b6ed840 100644 --- a/playbooks/roles/jump-vm/tasks/main.yaml +++ b/playbooks/roles/jump-vm/tasks/main.yaml @@ -92,7 +92,7 @@ - name: define jump VM command: "virt-install --connect qemu:///system --name {{ jumphost.name }} \ - --ram 4096 --vcpus=4 --os-type linux --os-variant ubuntu16.04 \ + --ram 8192 --vcpus=8 --os-type linux --os-variant ubuntu16.04 \ --disk path={{ workspace }}/kuberef-jump.qcow2,format=qcow2 \ --disk {{ workspace }}/kuberef-jump-cidata.iso,device=cdrom \ --network network=default,model=virtio,mac='{{ jumphost.interfaces[engine.net_config[engine.public_network].interface].mac_address }}' \ diff --git a/sw_config/bmra/dockerhub_credentials/tasks/main.yml b/sw_config/bmra/dockerhub_credentials/tasks/main.yml new file mode 100644 index 0000000..6531df8 --- /dev/null +++ b/sw_config/bmra/dockerhub_credentials/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: Add Docker hub credentials on target nodes + docker_login: + username: "{{ dhub_user }}" + password: "{{ dhub_pass }}" + email: "{{ dhub_email }}" + registry_url: "{{ dhub_reg_url }}" + no_log: True diff --git a/sw_config/bmra/dockerhub_credentials/vars/.gitignore b/sw_config/bmra/dockerhub_credentials/vars/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sw_config/bmra/dockerhub_credentials/vars/.gitignore diff --git a/sw_config/bmra/patched_k8s.yml b/sw_config/bmra/patched_k8s.yml new file mode 100644 index 0000000..52239b0 --- /dev/null +++ b/sw_config/bmra/patched_k8s.yml @@ -0,0 +1,166 @@ +## +## Copyright (c) 2020-2021 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- hosts: 127.0.0.1 + connection: local + tasks: [] + roles: + - { role: kubespray_install } + environment: "{{ proxy_env | d({}) }}" + any_errors_fatal: true + +- hosts: k8s-cluster + tasks: [] + roles: + - role: cluster_defaults + - role: kubespray_target_setup + environment: "{{ proxy_env | d({}) }}" + any_errors_fatal: true + +- hosts: all + gather_facts: false + tasks: + - name: prepare additional kubespray facts + set_fact: + kubelet_node_custom_flags_prepare: >- + {%- if native_cpu_manager_enabled | default(false) and native_cpu_manager_reserved_cpus is defined -%} + --reserved-cpus={{ native_cpu_manager_reserved_cpus }} + {%- endif -%} + enable_admission_plugins_prepare: >- + [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}] + kube_config_dir: /etc/kubernetes + - name: set kube_cert_dir + set_fact: + kube_cert_dir: "{{ kube_config_dir }}/ssl" + kube_csr_dir: "{{ kube_config_dir }}/csr" + environment: "{{ proxy_env | d({}) }}" + any_errors_fatal: true + +- name: run kubespray + import_playbook: kubespray/cluster.yml + vars: + kubeadm_enabled: true + multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf + docker_iptables_enabled: true + docker_dns_servers_strict: false + override_system_hostname: false + docker_version: '19.03' + kube_proxy_mode: iptables + enable_nodelocaldns: false + system_reserved: true + dashboard_enabled: true + system_cpu_reserved: "{{ native_cpu_manager_system_reserved_cpus | default('1000m') }}" + kube_cpu_reserved: "{{ native_cpu_manager_kube_reserved_cpus | default('1000m') }}" + kubelet_node_custom_flags: "{{ kubelet_node_custom_flags_prepare | from_yaml }}" + kube_api_anonymous_auth: true + kube_feature_gates: + - CPUManager=true # feature gate can be enabled by default, default policy is none in Kubernetes + - TopologyManager={{ topology_manager_enabled | default(true) }} + - RotateKubeletServerCertificate=true + # Kubernetes cluster hardening + kubernetes_audit: true + audit_log_maxbackups: 10 + kube_controller_manager_bind_address: 127.0.0.1 + kube_scheduler_bind_address: 127.0.0.1 + kube_proxy_healthz_bind_address: 127.0.0.1 + kube_proxy_metrics_bind_address: 127.0.0.1 + kube_read_only_port: 0 + kube_override_hostname: "" + kube_kubeadm_apiserver_extra_args: + service-account-lookup: true + service-account-key-file: "{{ kube_cert_dir }}/sa.key" + admission-control-config-file: "{{ kube_config_dir }}/admission-control/config.yaml" + kube_kubeadm_scheduler_extra_args: + address: 127.0.0.1 + profiling: false + kube_kubeadm_controller_extra_args: + address: 127.0.0.1 + service-account-private-key-file: "{{ kube_cert_dir }}/sa.key" + kubelet_config_extra_args: + protectKernelDefaults: true + cpuManagerPolicy: "{% if native_cpu_manager_enabled | default(false) %}static{% else %}none{% endif %}" + topologyManagerPolicy: "{{ topology_manager_policy | default('none') }}" + eventRecordQPS: 0 + kube_apiserver_request_timeout: 60s + kube_apiserver_enable_admission_plugins: "{{ enable_admission_plugins_prepare | from_yaml }}" + podsecuritypolicy_enabled: "{{ psp_enabled }}" + kube_encrypt_secret_data: true + apiserver_extra_volumes: + - name: admission-control-config + hostPath: /etc/kubernetes/admission-control/ + mountPath: /etc/kubernetes/admission-control/ + readOnly: true + preinstall_selinux_state: "{{ selinux_mode | default('disabled') }}" + tls_cipher_suites: + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + etcd_extra_vars: + ETCD_CIPHER_SUITES: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" + +- hosts: k8s-cluster + tasks: + - name: restart docker daemon to recreate iptables rules + systemd: name=docker state=restarted + become: yes + - name: restart kubelet to trigger static pods recreation + systemd: name=kubelet state=restarted + become: yes + # note: fix for the issue mentioned here: + # https://github.com/kubernetes-sigs/kubespray/blob/58f48500b1adac3f18466fa1c5cf8aa9d9838150/docs/flannel.md#flannel + - name: check if flannel.1 interface exists + stat: + path: /sys/class/net/flannel.1 + when: kube_network_plugin == "flannel" + register: flannel_endpoint + - name: disable offloading features on flannel.1 + command: ethtool --offload flannel.1 rx off tx off + become: yes + when: + - kube_network_plugin == "flannel" + - flannel_endpoint.stat.exists + +- hosts: etcd + tasks: + - name: change /var/lib/etcd owner + file: + path: "{{ etcd_data_dir | default('/var/lib/etcd') }}" + owner: etcd + group: etcd + recurse: true + state: directory + mode: 0700 + - name: change /var/lib/etcd permissions + file: + path: "{{ etcd_data_dir | default('/var/lib/etcd') }}" + owner: etcd + group: etcd + mode: '0700' + state: directory + +- hosts: k8s-cluster + roles: + - role: cluster_defaults + tags: defaults + - role: docker_registry + tags: registry + - role: dockerhub_credentials + when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file" + environment: "{{ proxy_env | d({}) }}" + any_errors_fatal: true + +- name: run certificate generation for mTLS in kubelet + import_playbook: kubelet-certificates.yml |