From 58fae5e1bdc4e322ebd006f0b940d43b70d0d1c3 Mon Sep 17 00:00:00 2001 From: Adrien Date: Thu, 17 Sep 2020 01:15:23 +0200 Subject: [PATCH] Update kubelet config --- tasks/cluster_kubeadm.yml | 38 +++++++++++++++ tasks/install_server.yml | 46 +++++++++++++------ templates/etc/sysconfig/kubelet.j2 | 2 +- .../0-kubelet-extra-args.conf.j2 | 2 +- templates/kubeadm-config.yaml.j2 | 23 +++++++++- vars/RedHat.yml | 1 + 6 files changed, 96 insertions(+), 16 deletions(-) diff --git a/tasks/cluster_kubeadm.yml b/tasks/cluster_kubeadm.yml index bf9a114..fb037b2 100644 --- a/tasks/cluster_kubeadm.yml +++ b/tasks/cluster_kubeadm.yml @@ -44,6 +44,44 @@ when: - kubernetes_kubeproxy_mode == "ipvs" +- name: Create thin volumes for kubernetes + lvol: + vg: "{{ item.vg }}" + lv: "{{ item.name }}" + thinpool: kubernetes + size: "{{ item.size }}" + with_items: + - { name: var_lib_etcd, vg: vg_sys, size: 1g, mount_point: /var/lib/etcd, mount_opts: "nodev,noexec,nosuid,discard"} + - { name: etc_kubernetes, vg: vg_sys, size: 1g, mount_point: /etc/kubernetes, mount_opts: "nodev,noexec,nosuid,discard"} +# - { name: var_lib_kubelet, vg: vg_sys, size: 128m, mount_point: /var/lib/kubelet, mount_opts: "discard"} + when: + - kubernetes_master|bool + +- name: create file system on containerd lv + filesystem: + fstype: ext4 + dev: "/dev/{{ item.vg }}/{{ item.name }}" + with_items: + - { name: var_lib_etcd, vg: vg_sys, size: 1g, mount_point: /var/lib/etcd, mount_opts: "nodev,noexec,nosuid,discard"} + - { name: etc_kubernetes, vg: vg_sys, size: 1g, mount_point: /etc/kubernetes, mount_opts: "nodev,noexec,nosuid,discard"} +# - { name: var_lib_kubelet, vg: vg_sys, size: 128m, mount_point: /var/lib/kubelet, mount_opts: "discard"} + when: + - kubernetes_master|bool + +- name: mount logical volumes + mount: + name: "{{ item.mount_point }}" + src: "/dev/{{ item.vg }}/{{ item.name }}" + fstype: ext4 + opts: "{{ item.mount_opts }}" + state: mounted + with_items: + - { name: var_lib_etcd, vg: vg_sys, size: 1g, mount_point: /var/lib/etcd, mount_opts: "nodev,noexec,nosuid,discard"} + - { name: etc_kubernetes, vg: vg_sys, size: 1g, mount_point: /etc/kubernetes, mount_opts: "nodev,noexec,nosuid,discard"} +# - { name: var_lib_kubelet, vg: vg_sys, size: 128m, mount_point: /var/lib/kubelet, mount_opts: "discard"} + when: + - kubernetes_master|bool + - name: Secure etcd directory file: path: "/var/lib/etcd" diff --git a/tasks/install_server.yml b/tasks/install_server.yml index 50a487d..7fb8a12 100644 --- a/tasks/install_server.yml +++ b/tasks/install_server.yml @@ -9,6 +9,12 @@ fstype: swap state: absent +- name: Create a thin pool for kubernetes + lvol: + vg: vg_sys + thinpool: kubernetes + size: 20g + # Install API loadbalancer - include_tasks: "load_balancer.yml" when: @@ -26,7 +32,7 @@ - kubernetes_master|bool - name: Configure audit policy - file: + copy: src: "etc/kubernetes/policies/audit-policy.yaml" dest: "/etc/kubernetes/policies/audit-policy.yaml" group: root @@ -44,16 +50,6 @@ when: - kubernetes_cri == "k3s" -- name: Check if a node is still tainted - command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes '{{ ansible_hostname | lower }}' -o jsonpath='{.spec.taints}' - when: kubernetes_master_taint - register: current_taint - -- name: taint the machine if needed -# command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master- - command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes '{{ ansible_hostname | lower }}' node-role.kubernetes.io/master- - when: kubernetes_master_taint|bool and current_taint.stdout - # # At this point, we have a kubernetes up and running, but ready for it # @@ -68,9 +64,9 @@ when: - kubernetes_master|bool -- name: Copy certificat file on +- name: Copy kubeconfig file from /etc/kubernetes/admin.conf copy: - src: /etc/kubernetes/admin.conf + src: "/etc/kubernetes/admin.conf" dest: /root/.kube/config remote_src: yes owner: root @@ -78,6 +74,19 @@ mode: 0600 when: - kubernetes_master|bool + - kubernetes_cri != "k3s" + +- name: Copy kubeconfig file from /etc/rancher/k3s/k3s.yaml + copy: + src: "/etc/rancher/k3s/k3s.yaml" + dest: /root/.kube/config + remote_src: yes + owner: root + group: root + mode: 0600 + when: + - kubernetes_master|bool + - kubernetes_cri == "k3s" # # Manque autoconfig de .kube/config local @@ -89,3 +98,14 @@ # dest: /root/.kube/{{ kubernetes_cluster_name }}/ca.crt # when: # - kubernetes_master|bigip_pool + +- name: Check if a node is still tainted + command: kubectl get nodes '{{ ansible_hostname | lower }}' -o jsonpath='{.spec.taints}' + when: kubernetes_master_taint + register: current_taint + +- name: taint the machine if needed +# command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master- + command: kubectl taint nodes '{{ ansible_hostname | lower }}' node-role.kubernetes.io/master- + when: kubernetes_master_taint|bool and current_taint.stdout + diff --git a/templates/etc/sysconfig/kubelet.j2 b/templates/etc/sysconfig/kubelet.j2 index 8d34dfb..f7d16d3 100644 --- a/templates/etc/sysconfig/kubelet.j2 +++ b/templates/etc/sysconfig/kubelet.j2 @@ -1,2 +1,2 @@ #https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates -KUBELET_EXTRA_ARGS="{% if ansible_service_mgr == "systemd" %}--cgroup-driver=systemd {% endif %}--container-runtime=remote --runtime-request-timeout=5m --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_eth0.ipv4.address }}" +KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_eth0.ipv4.address }}" diff --git a/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 b/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 index 6325b64..7c704b9 100644 --- a/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 +++ b/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 @@ -1,2 +1,2 @@ [Service] -Environment=KUBELET_EXTRA_ARGS="--cgroup-driver=systemd --container-runtime=remote --runtime-request-timeout=5m --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_eth0.ipv4.address }}" +Environment=KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_eth0.ipv4.address }}" diff --git a/templates/kubeadm-config.yaml.j2 b/templates/kubeadm-config.yaml.j2 index e98a1e3..2d90758 100644 --- a/templates/kubeadm-config.yaml.j2 +++ b/templates/kubeadm-config.yaml.j2 @@ -42,7 +42,7 @@ nodeRegistration: localAPIEndpoint: advertiseAddress: "{{ ansible_default_ipv4.address }}" bindPort: 6443 -{% if kubernetes_master|bool %} +{% if kubernetes_master|bool and groups['KubernetesMasterConfigured'] is defined %} certificateKey: "{{ kubernetes_certificateKey.stdout }}" {% endif %} --- @@ -53,14 +53,18 @@ controlPlane: localAPIEndpoint: advertiseAddress: "{{ ansible_default_ipv4.address }}" bindPort: 6443 +{% if groups['KubernetesMasterConfigured'] is defined %} certificateKey: "{{ kubernetes_certificateKey.stdout }}" {% endif %} +{% endif %} discovery: bootstrapToken: apiServerEndpoint: "{{ lb_kubemaster }}:6443" +{% if groups['KubernetesMasterConfigured'] is defined %} caCertHashes: - sha256:{{ cacerthash.stdout }} token: "{{ kubetoken.stdout }}" +{% endif %} nodeRegistration: kubeletExtraArgs: node-ip: {{ ansible_default_ipv4.address }} @@ -117,3 +121,20 @@ mode: {{ kubernetes_kubeproxy_mode }} --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration +runtimeRequestTimeout: 5m +{% if ansible_service_mgr == "systemd" %} +cgroupDriver: systemd +{% endif %} + +{% if false %} +readOnlyPort: 1 +systemReserved: + cpu=200m,memory=200M + +containerRuntime: remote +{% if kubernetes_cri == "containerd" %} +containerRuntimeEndpoint: "unix:///run/containerd/containerd.sock" +{% elif kubernetes_cri == "cri-o" %} +containerRuntimeEndpoint: "unix:///var/run/crio/crio.sock" +{% endif %} +{% endif %} diff --git a/vars/RedHat.yml b/vars/RedHat.yml index f75fc30..a5905f3 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -3,5 +3,6 @@ kubernetes_package_name: - kubectl - kubelet - kubeadm + - iproute-tc #kubernetes_remove_packages_name: # - kubernetes.io