Update kubernetes to versoin 1.31.5

Reviewed-on: #1

README.md

@@ -9,3 +9,5 @@ Deploy kubernetes
 
 https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2?tab=doc
 https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/
+
+https://github.com/XenitAB/spegel

defaults/main.yml

@@ -2,7 +2,16 @@
 # value for kubernetes_cri: containerd, cri-o
 #kubernetes_cri: "containerd"
 kubernetes_server: false
+kubernetes_interface: '{{ ansible_default_ipv4.interface }}'
 # value for kuberntes_network: flannel, calico, weave-net
 #kubernetes_network: weave-net
 kubernetes_kubeproxy_mode: ipvs
-kubernetes_version: 1.20.1
+kubernetes_version: 1.31.5
+kubernetes_k3s_version: 1.31.5+k3s1
+#kubernetes_pods_network: "10.244.0.0/16"
+#kubernetes_svc_network: "10.96.0.0/12"
+kubernetes_pods_network: "10.42.0.0/16"
+kubernetes_svc_network: "10.43.0.0/16"
+lb_auth_pass: 1be344d62acc46c6858ae8475668a245
+kubernetes_swap_enabled: false
+kubernetes_lvm: true

files/etc/NetworkManager/conf.d/calico.conf

@@ -0,0 +1,3 @@
+# https://docs.tigera.io/calico/latest/operations/troubleshoot/troubleshooting#configure-networkmanager
+[keyfile]
+unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:wireguard.cali

files/etc/kubernetes/policies/audit-policy.yaml

@@ -10,7 +10,15 @@ rules:
     - group: ""
       # Resource "pods" doesn't match requests to any subresource of pods,
       # which is consistent with the RBAC policy.
-      resources: ["pods"]
+      resources: ["pods", "deployments"]
+
+  - level: RequestResponse
+    resources:
+    - group: "rbac.authorization.k8s.io"
+      # Resource "pods" doesn't match requests to any subresource of pods,
+      # which is consistent with the RBAC policy.
+      resources: ["clusterroles", "clusterrolebindings"]
+
   # Log "pods/log", "pods/status" at Metadata level
   - level: Metadata
     resources:
@@ -48,11 +56,17 @@ rules:
     # The empty string "" can be used to select non-namespaced resources.
     namespaces: ["kube-system"]
 
-  # Log configmap and secret changes in all other namespaces at the Metadata level.
+  # Log configmap changes in all other namespaces at the RequestResponse level.
+  - level: RequestResponse
+    resources:
+    - group: "" # core API group
+      resources: ["configmaps"]
+
+  # Log secret changes in all other namespaces at the Metadata level.
   - level: Metadata
     resources:
     - group: "" # core API group
-      resources: ["secrets", "configmaps"]
+      resources: ["secrets"]
 
   # Log all other resources in core and extensions at the Request level.
   - level: Request

files/etc/kubernetes/psa.yaml

@@ -0,0 +1,27 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: AdmissionConfiguration
+plugins:
+- name: PodSecurity
+  configuration:
+    apiVersion: pod-security.admission.config.k8s.io/v1beta1
+    kind: PodSecurityConfiguration
+    defaults:
+      enforce: "restricted"
+      enforce-version: "latest"
+      audit: "restricted"
+      audit-version: "latest"
+      warn: "restricted"
+      warn-version: "latest"
+    exemptions:
+      usernames: []
+      runtimeClasses: []
+      namespaces: [kube-system, cis-operator-system]
+- name: EventRateLimit
+  configuration:
+    apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+    kind: Configuration
+    limits:
+    - burst: 20000
+      qps: 5000
+      type: Server
+  path: ""

meta/main.yml

@@ -10,7 +10,9 @@ galaxy_info:
       version:
         - 7
         - 8
+        - 9
     - name: RedHat
       version:
         - 7
         - 8
+        - 9

tasks/Debian.yml

@@ -1,21 +1,21 @@
 ---
-- name: add docker apt key
+- name: Add docker apt key
-  apt_key:
+  ansible.builtin.apt_key:
     url: https://download.docker.com/linux/ubuntu/gpg
     state: present
   when:
     - docker_ver == "docker_ce"
 
-- name: add docker repository
+- name: Add docker repository
-  apt_repository:
+  ansible.builtin.apt_repository:
     repo: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable'
     state: present
-    update_cache: yes
+    update_cache: true
   when:
     - docker_ver == "docker_ce"
 
 - name: "Ensure GRUB_CMDLINE_LINUX is updated"
-  lineinfile:
+  ansible.builtin.lineinfile:
     dest: /etc/default/grub
     regexp: '^(GRUB_CMDLINE_LINUX=".*)"$'
     line: '\1 cgroup_enable=memory swapaccount=1"'
@@ -24,12 +24,12 @@
     - not docker_installed.stat.exists
 
 - name: "Update grub.conf"
-  command: update-grub
+  ansible.builtin.command: update-grub
   when:
     - not docker_installed.stat.exists
 
 - name: "Ensure DEFAULT_FORWARD_POLICY in /etc/default/ufw is updated"
-  lineinfile:
+  ansible.builtin.lineinfile:
     dest: /etc/default/ufw
     regexp: '^(DEFAULT_FORWARD_POLICY=").*"$'
     line: '\1ACCEPT"'

tasks/RedHat.yml

@@ -5,55 +5,155 @@
 #     description: "Kubernetes Repository"
 #     baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-$releasever-x86_64
 #     gpgcheck: yes
-#    enabled: yes
+#     enabled: true
 #     gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg
 #     state: present
 
-- name: Add Official kubernetes's repo
+# - name: Add Official kubernetes's repo
-  template:
+#   ansible.builtin.template:
-    src: "etc/yum.repos.d/kubernetes.repo.j2"
+#     src: "etc/yum.repos.d/kubernetes.repo.j2"
-    dest: "/etc/yum.repos.d/kubernetes.repo"
+#     dest: "/etc/yum.repos.d/kubernetes.repo"
-    group: root
+#     group: root
-    owner: root
+#     owner: root
-    mode: 0644
+#     mode: 0644
+#   when:
+#     - not ansible_machine == "armv7l"
+#     - not ansible_machine == "armv6l"
+#     - kubernetes_cri != "k3s"
+
+- name: Add Official kubernetes's repo on servers
+  ansible.builtin.yum_repository:
+    name: kubernetes
+    description: Kubernetes
+    baseurl: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/"
+    enabled: true
+    gpgcheck: true
+    repo_gpgcheck: true
+    gpgkey: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/repodata/repomd.xml.key"
+    exclude: kubelet kubeadm
+  become: true
   when:
     - not ansible_machine == "armv7l"
     - not ansible_machine == "armv6l"
+    - kubernetes_server|bool
     - kubernetes_cri != "k3s"
 
-- name: Register kubernetes firewalld service
+- name: Add Official kubernetes's repo for Desktop
-  template:
+  ansible.builtin.yum_repository:
-    src: "etc/firewalld/services/kubernetes.xml.j2"
+    name: kubernetes
-    dest: "/etc/firewalld/services/kubernetes.xml"
+    description: Kubernetes
-    group: root
+    baseurl: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/"
-    owner: root
+    enabled: true
-    mode: 0644
+    gpgcheck: true
-  register: need_firewalld_reload
+    repo_gpgcheck: true
+    gpgkey: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/repodata/repomd.xml.key"
+    exclude: kubelet kubeadm kubectl
+  become: true
   when:
-    - kubernetes_server|bool
+    - not ansible_machine == "armv7l"
+    - not ansible_machine == "armv6l"
+    - not kubernetes_server|bool
 
-#- name: Reload firewalld configuration
+# - name: Redhat | Installing K8s Packages
-#  service:
+#   ansible.builtin.package:
-#    name: firewalld
+#     name:
-#    state: reloaded
+#       - kubectl
-#    enabled: yes
+#       - kubelet
+#       - kubeadm
+#       - iproute-tc
+#       - ipvsadm
+#     state: present
+#     disable_excludes: kubernetes
+#   become: true
+#   register: result
+#   until: result is successful
+
+# - name: Register kubernetes firewalld service
+#   ansible.builtin.template:
+#     src: "etc/firewalld/services/kubernetes.xml.j2"
+#     dest: "/etc/firewalld/services/kubernetes.xml"
+#     group: root
+#     owner: root
+#     mode: 0644
+#   register: need_firewalld_reload
 #   when:
 #     - kubernetes_server|bool
+# 
+# - name: Reload firewalld configuration
+#   ansible.builtin.service:
+#     name: firewalld
+#     state: reloaded
+#     enabled: true
+#   when:
+#     - kubernetes_server|bool
+#     - need_firewalld_reload is changed
+# 
+##  Définir interface
+# - name: Open Firewalld
+#   ansible.posix.firewalld:
+#     zone: external
+#     service: kubernetes
+#     permanent: true
+#     state: enabled
+#     immediate: true
+#   when:
+##     - need_firewall|bool
+##     - firewall_name == "firewalld"
+#     - kubernetes_server|bool
 
-- name: reload firewalld to refresh service list
+# - name: Create kubernetes firewalld zone
-  command: firewall-cmd --reload
+#   ansible.posix.firewalld:
-  when:
+#     zone: kubernetes
-    - need_firewalld_reload is changed
+#     permanent: true
-    - kubernetes_server|bool
+#     state: present
-
+#   when:
-# Définir interface
+#     - kubernetes_server|bool
-- name: Open Firewalld
+- name: Add kubernetes networks to trusted firewalld zone
-  firewalld:
+  ansible.posix.firewalld:
-    service: kubernetes
+#    zone: kubernetes
+    zone: trusted
     permanent: true
     state: enabled
-    immediate: true
+    source: "{{ item }}"
   when:
-#    - need_firewall|bool
-#    - firewall_name == "firewalld"
     - kubernetes_server|bool
+  with_items:
+    - "{{ kubernetes_pods_network }}"
+    - "{{ kubernetes_svc_network }}"
+
+- name: Add kubernetes networks to trusted firewalld zone
+  ansible.posix.firewalld:
+#    zone: kubernetes
+    zone: trusted
+    permanent: true
+    state: enabled
+    source: "{{ item }}"
+  when:
+    - kubernetes_server|bool
+    - kubernetes_interface is defined
+#    - false
+  with_items:
+    - "{{ (lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.network + '/' + lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.netmask) | ansible.utils.ipaddr('net') }}"
+
+- name: Install kubernetes tools
+  ansible.builtin.dnf:
+    name: "{{ kubernetes_package_name }}"
+    enablerepo: "kubernetes"
+    state: present
+    update_cache: true
+    disable_excludes: kubernetes
+#  notify: Restart kubelet
+  when:
+    - ansible_pkg_mgr == "dnf"
+    - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s")
+
+- name: Install kubernetes tools
+  ansible.builtin.yum:
+    name: "{{ kubernetes_package_name }}"
+    enablerepo: "kubernetes"
+    state: present
+    update_cache: true
+#  notify: Restart kubelet
+  when:
+    - ansible_pkg_mgr == "yum"
+    - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s")

tasks/cluster_k3s.yml

@@ -1,19 +1,46 @@
 ---
 - name: Install Wireguard
-  include_role:
+  ansible.builtin.include_role:
     name: wireguard
-#  when:
+  when:
 #    - kubernetes_cni == "wireguard"
+    - "'Vpn' not in group_names"
+
+- name: Import Rancher key
+  ansible.builtin.rpm_key:
+    state: present
+    key: https://rpm.rancher.io/public.key
+  when:
+    - ansible_os_family == "RedHat"
+
+- name: Install the k3s-selinux rpm from a remote repo for yum distro
+  ansible.builtin.yum:
+    name: "https://github.com/k3s-io/k3s-selinux/releases/download/v1.6.stable.1/k3s-selinux-1.6-1.el7.noarch.rpm"
+    state: present
+  when:
+    - ansible_pkg_mgr == "yum"
+    - ansible_os_family == "RedHat"
+    - ansible_distribution_major_version == '7'
+
+- name: Install the k3s-selinux rpm from a remote repo for dnf distro
+  ansible.builtin.dnf:
+    name: "https://github.com/k3s-io/k3s-selinux/releases/download/v1.6.stable.1/k3s-selinux-1.6-1.el{{ ansible_distribution_major_version }}.noarch.rpm"
+    state: present
+  when:
+    - ansible_pkg_mgr == "dnf"
+    - ansible_os_family == "RedHat"
+    - ansible_distribution_major_version >= '8'
 
 - name: Check if /usr/local/bin/k3s already existe
-  stat:
+  ansible.builtin.stat:
     path: /usr/local/bin/k3s
   register: k3s_bin
+  check_mode: false
   changed_when: False
 
-- name: retreive k3s binary for x86_64
+- name: Retreive k3s binary for x86_64
-  get_url:
+  ansible.builtin.get_url:
-    url: "https://github.com/rancher/k3s/releases/download/v1.20.2%2Bk3s1/k3s"
+    url: "https://github.com/rancher/k3s/releases/download/v{{ kubernetes_k3s_version | urlencode }}/k3s"
     dest: "/usr/local/bin/k3s"
     group: root
     owner: root
@@ -22,9 +49,9 @@
     - not k3s_bin.stat.exists
     - ansible_machine == "x86_64"
 
-- name: retreive k3s binary for arm64
+- name: Retreive k3s binary for arm64
-  get_url:
+  ansible.builtin.get_url:
-    url: "https://github.com/rancher/k3s/releases/download/v1.20.2%2Bk3s1/k3s-arm64"
+    url: "https://github.com/rancher/k3s/releases/download/v{{ kubernetes_k3s_version | urlencode }}/k3s-arm64"
     dest: "/usr/local/bin/k3s"
     group: root
     owner: root
@@ -33,9 +60,9 @@
     - not k3s_bin.stat.exists
     - ansible_machine == "arm64"
 
-- name: retreive k3s binary for armv6/armv7
+- name: Retreive k3s binary for armv6/armv7
-  get_url:
+  ansible.builtin.get_url:
-    url: "https://github.com/rancher/k3s/releases/download/v1.20.2%2Bk3s1/k3s-armhf"
+    url: "https://github.com/rancher/k3s/releases/download/v{{ kubernetes_k3s_version | urlencode }}/k3s-armhf"
     dest: "/usr/local/bin/k3s"
     group: root
     owner: root
@@ -45,7 +72,7 @@
     - (ansible_machine == "armv7l") or (ansible_machine == "armv6l")
 
 - name: Create tools link
-  file:
+  ansible.builtin.file:
     src: "k3s"
     dest: "/usr/local/bin/{{ item }}"
     owner: root
@@ -56,23 +83,12 @@
     - "crictl"
     - "ctr"
 
-# Manque kubernetes_server_token, kubernetes_master url
+- name: Create logical volume for k3s
-
-- name: Deploy systemd service
-  template:
-    src: "etc/systemd/system/{{ item }}.j2"
-    dest: "/etc/systemd/system/{{ item }}"
-    owner: root
-    group: root
-    mode: 0600
-  with_items:
-    - "k3s.service"
-    - "k3s.service.env"
   when:
-    - ansible_service_mgr == "systemd"
+    - kubernetes_lvm|bool
-
+  block:
   - name: Create thin volumes for k3s
-  lvol:
+    community.general.lvol:
       vg: "{{ item.vg }}"
       lv: "{{ item.name }}"
       thinpool: kubernetes
@@ -80,15 +96,15 @@
     with_items:
       - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"}
 
-- name: create file system on containerd lv
+  - name: Create file system on containerd lv
-  filesystem:
+    community.general.filesystem:
       fstype: ext4
       dev: "/dev/{{ item.vg }}/{{ item.name }}"
     with_items:
       - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"}
 
-- name: mount logical volumes
+  - name: Mount logical volumes
-  mount:
+    ansible.posix.mount:
       name: "{{ item.mount_point }}"
       src: "/dev/{{ item.vg }}/{{ item.name }}"
       fstype: ext4
@@ -97,8 +113,195 @@
     with_items:
       - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"}
 
+- name: Ensure protect-kernel-defaults is set
+  ansible.posix.sysctl:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    sysctl_file: /etc/sysctl.d/90-kubelet.conf
+    reload: true
+  with_items:
+    - { name: "vm.panic_on_oom", value: "0" }
+    - { name: "vm.overcommit_memory", value: "1" }
+    - { name: "kernel.panic", value: "10" }
+    - { name: "kernel.panic_on_oops", value: "1" }
+  when:
+    - kubernetes_server|bool
+
+- name: /etc/kubernetes directory
+  ansible.builtin.file:
+    path: "/etc/kubernetes"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  when:
+    - kubernetes_master|bool
+
+- name: Configure Pod Security
+  ansible.builtin.copy:
+    src: "etc/kubernetes/psa.yaml"
+    dest: "/etc/kubernetes/psa.yaml"
+    group: root
+    owner: root
+    mode: 0644
+  when:
+    - kubernetes_master|bool
+
+- name: Audit policies directory
+  ansible.builtin.file:
+    path: "/etc/kubernetes/policies"
+    state: directory
+    owner: root
+    group: root
+    mode: 0700
+  when:
+    - kubernetes_master|bool
+
+- name: Configure audit policy
+  ansible.builtin.copy:
+    src: "etc/kubernetes/policies/audit-policy.yaml"
+    dest: "/etc/kubernetes/policies/audit-policy.yaml"
+    group: root
+    owner: root
+    mode: 0644
+  when:
+    - kubernetes_master|bool
+
+# Check controlers
+- name: Check if /etc/rancher/k3s/k3s.yaml already existe
+  ansible.builtin.stat:
+    path: /etc/rancher/k3s/k3s.yaml
+  register: st
+  check_mode: false
+  changed_when: False
+  when:
+    - kubernetes_master|bool
+
+- name: Create KubernetesMasterConfigured group
+  ansible.builtin.group_by:
+    key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }}
+  check_mode: false
+  when:
+    - kubernetes_master|bool
+    - st.stat.exists
+
+# First controler
+- name: Configure first controler
+#  run_once: true
+  block:
+  - name: Create k3s directories on master nodes
+    ansible.builtin.file:
+      path: "{{ item }}"
+      state: directory
+      owner: root
+      group: root
+      mode: 0700
+    with_items:
+      - "/etc/rancher"
+      - "/etc/rancher/k3s"
+      - "/etc/rancher/k3s/config.yaml.d"
+      - "/var/lib/rancher"
+      - "/var/lib/rancher/k3s"
+      - "/var/lib/rancher/k3s/server"
+      - "/var/lib/rancher/k3s/server/manifests"
+    when:
+      - kubernetes_master|bool
+
+  - name: Create k3s directories on all nodes
+    ansible.builtin.file:
+      path: "{{ item }}"
+      state: directory
+      owner: root
+      group: root
+      mode: 0700
+    with_items:
+      - "/var/lib/rancher/k3s/storage"
+
+# semanage fcontext -a -t container_file_t  "/var/lib/rancher/k3s/storage(/.*)?"
+  - name: Allow K3S local-path provisioner to create directories in /var/lib/rancher/k3s/storage
+    community.general.sefcontext:
+      target: '/var/lib/rancher/k3s/storage(/.*)?'
+      setype: container_file_t
+      state: present
+  - name: Apply new SELinux file context to filesystem
+    ansible.builtin.command: restorecon -R  /var/lib/rancher/k3s/storage/
+
+  - name: Deploy Network Policies
+    ansible.builtin.template:
+      src: "{{ item }}.j2"
+      dest: "/{{ item }}"
+      owner: root
+      group: root
+      mode: 0600
+    with_items:
+      - "var/lib/rancher/k3s/server/manifests/np-00-intra-namespace.yaml"
+      - "var/lib/rancher/k3s/server/manifests/np-01-default-network-dns-policy.yaml"
+      - "var/lib/rancher/k3s/server/manifests/np-03-metrics-server-traefik.yaml"
+    when:
+      - kubernetes_master|bool
+
+  - name: Deploy systemd service
+    ansible.builtin.template:
+      src: "{{ item }}.j2"
+      dest: "/{{ item }}"
+      owner: root
+      group: root
+      mode: 0600
+    with_items:
+      - "etc/systemd/system/k3s.service"
+      - "etc/systemd/system/k3s.service.env"
+      - "etc/rancher/k3s/config.yaml"
+    when:
+      - ansible_service_mgr == "systemd"
+
+  - name: Reload systemd
+    ansible.builtin.systemd:
+      daemon_reload: true
+
   - name: Enable k3s on boot
-  service:
+    ansible.builtin.service:
       name: k3s
       state: started
-    enabled: yes
+      enabled: true
+
+  - name: Wait for k3s.yaml
+    wait_for:
+      path: /etc/rancher/k3s/k3s.yaml
+
+  - name: Wait for token
+    wait_for:
+      path: /var/lib/rancher/k3s/server/token
+
+  - name: Add {{ ansible_hostname }} to KubernetesMasterConfigured group
+    ansible.builtin.group_by:
+      key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }}
+    check_mode: false
+
+  when:
+    - kubernetes_master|bool
+    - vars['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined
+
+# chmod -R 600 /var/lib/rancher/k3s/server/tls/*.crt
+
+# Manque kubernetes_server_token, kubernetes_master url
+
+# - name: Deploy systemd service
+#   ansible.builtin.template:
+#     src: "etc/systemd/system/{{ item }}.j2"
+#     dest: "/etc/systemd/system/{{ item }}"
+#     owner: root
+#     group: root
+#     mode: 0600
+#   with_items:
+#     - "k3s.service"
+#     - "k3s.service.env"
+#   when:
+#     - ansible_service_mgr == "systemd"
+
+
+
+- name: Enable k3s on boot
+  ansible.builtin.service:
+    name: k3s
+    state: started
+    enabled: true

tasks/cluster_kubeadm.yml

@@ -1,27 +1,46 @@
 ---
 - name: Install Containerd
-  include_role:
+  ansible.builtin.include_role:
     name: containerd
   when:
     - kubernetes_cri == "containerd"
   # register: kubernetes_cri_changed
 
 - name: Install CRI-O
-  include_role:
+  ansible.builtin.include_role:
     name: cri-o
   when:
     - kubernetes_cri == "cri-o"
   # register: kubernetes_cri_changed
 
 # - name: Restart kubelet after kubernetes cri installation
-#  service:
+#   ansible.builtin.service:
 #     name: kubelet
 #     status: restarted
 #   when:
 #     - kubernetes_cri_changed is changed
 
+- name: Configure NetworkManager for Calico
+  ansible.builtin.copy:
+    src: "etc/NetworkManager/conf.d/calico.conf"
+    dest: "/etc/NetworkManager/conf.d/calico.conf"
+    group: root
+    owner: root
+    mode: 0644
+  when:
+    - kubernetes_network == "calico"
+    - ansible_os_family == "RedHat"
+  register: kubernetes_network_networkmanager_changed
+
+- name: Restart kubelet after kubernetes cri installation
+  ansible.builtin.service:
+    name: NetworkManager
+    status: reload
+  when:
+    - kubernetes_network_networkmanager_changed is changed
+
 - name: Configuring IPVS kernel module to be load on boot
-  template:
+  ansible.builtin.template:
     src: "etc/modules-load.d/ipvs.conf.j2"
     dest: "/etc/modules-load.d/ipvs.conf"
     group: root
@@ -31,7 +50,7 @@
     - kubernetes_kubeproxy_mode == "ipvs"
 
 - name: Load IPVS kernel module for EL7
-  modprobe:
+  community.general.modprobe:
     name: "{{ item }}"
     state: present
   with_items:
@@ -47,7 +66,7 @@
     - ansible_distribution_major_version == '7'
 
 - name: Load IPVS kernel module for EL8
-  modprobe:
+  community.general.modprobe:
     name: "{{ item }}"
     state: present
   with_items:
@@ -59,10 +78,10 @@
   when:
     - kubernetes_kubeproxy_mode == "ipvs"
     - ansible_os_family == "RedHat"
-    - ansible_distribution_major_version == '8'
+    - ansible_distribution_major_version >= '8'
 
 - name: Create thin volumes for kubernetes
-  lvol:
+  community.general.lvol:
     vg: "{{ item.vg }}"
     lv: "{{ item.name }}"
     thinpool: kubernetes
@@ -74,8 +93,8 @@
   when:
     - kubernetes_master|bool
 
-- name: create file system on containerd lv
+- name: Create file system on containerd lv
-  filesystem:
+  community.general.filesystem:
     fstype: ext4
     dev: "/dev/{{ item.vg }}/{{ item.name }}"
   with_items:
@@ -85,8 +104,8 @@
   when:
     - kubernetes_master|bool
 
-- name: mount logical volumes
+- name: Mount logical volumes
-  mount:
+  ansible.posix.mount:
     name: "{{ item.mount_point }}"
     src: "/dev/{{ item.vg }}/{{ item.name }}"
     fstype: ext4
@@ -101,14 +120,14 @@
     - kubernetes_master|bool
 
 - name: Ensuring /var/lib/etcd/lost+found Folder does not exists
-  file:
+  ansible.builtin.file:
     path: "/var/lib/etcd/lost+found"
     state: "absent"
   when:
     - partition_formated is changed
 
 - name: Secure etcd directory
-  file:
+  ansible.builtin.file:
     path: "/var/lib/etcd"
     state: directory
     owner: root
@@ -118,7 +137,7 @@
     - kubernetes_master|bool
 
 - name: Ensuring /etc/systemd/system/kubelet.service.d Folder Exists
-  file:
+  ansible.builtin.file:
     path: "/etc/systemd/system/kubelet.service.d"
     state: "directory"
     group: root
@@ -128,7 +147,7 @@
     - ansible_service_mgr == "systemd"
 
 - name: Configure kubelet service
-  template:
+  ansible.builtin.template:
     src: "etc/{{ item }}.j2"
     dest: "/etc/{{ item }}"
     group: root
@@ -140,8 +159,35 @@
   when:
     - ansible_service_mgr == "systemd"
 
+- name: Configure kubelet service for CRI-O
+  ansible.builtin.template:
+    src: "etc/{{ item }}.j2"
+    dest: "/etc/{{ item }}"
+    group: root
+    owner: root
+    mode: 0644
+  with_items:
+    - "systemd/system/kubelet.service.d/11-cgroups.conf"
+  when:
+    - ansible_service_mgr == "systemd"
+    - kubernetes_cri == "cri-o"
+
+- name: Enable Swap for kubelet service
+  ansible.builtin.template:
+    src: "etc/{{ item }}.j2"
+    dest: "/etc/{{ item }}"
+    group: root
+    owner: root
+    mode: 0644
+  with_items:
+    - "systemd/system/kubelet.service.d/20-allow-swap.conf"
+  when:
+    - ansible_service_mgr == "systemd"
+    - kubernetes_swap_enabled is defined
+    - kubernetes_swap_enabled|bool
+
 - name: Configure kubelet service
-  template:
+  ansible.builtin.template:
     src: "etc/{{ item }}.j2"
     dest: "/etc/{{ item }}"
     group: root
@@ -153,79 +199,105 @@
     - not ansible_service_mgr == "systemd"
 
 - name: Enable kubelet on boot
-  service:
+  ansible.builtin.service:
     name: kubelet
     state: started
-    enabled: yes
+    enabled: true
+
+- name: Audit policies directory
+  ansible.builtin.file:
+    path: "/etc/kubernetes/policies"
+    state: directory
+    owner: root
+    group: root
+    mode: 0700
+  when:
+    - kubernetes_master|bool
+
+# https://v1-17.docs.kubernetes.io/docs/tasks/debug-application-cluster/falco/
+# https://github.com/falcosecurity/falco/blob/master/rules/k8s_audit_rules.yaml
+# Ou récupération de ces règles pour une utilisation avec falco
+
+- name: Configure audit policy
+  ansible.builtin.copy:
+    src: "etc/kubernetes/policies/audit-policy.yaml"
+    dest: "/etc/kubernetes/policies/audit-policy.yaml"
+    group: root
+    owner: root
+    mode: 0644
+  when:
+    - kubernetes_master|bool
 
 # First controler
 - name: Check if /etc/kubernetes/admin.conf already existe
-  stat:
+  ansible.builtin.stat:
     path: /etc/kubernetes/admin.conf
   register: st
+  check_mode: false
   changed_when: False
 
 - name: Create KubernetesMasterConfigured group
-  group_by:
+  ansible.builtin.group_by:
-    key: KubernetesMasterConfigured
+    key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }}
+  check_mode: false
   when:
     - st.stat.exists
 
 - name: Retreive kubeadm Major version
-  shell: set -o pipefail && kubeadm version | sed 's/.*{Major:"\([0-9]\)".*/\1/'
+  ansible.builtin.shell: set -o pipefail && kubeadm version | sed 's/.*{Major:"\([0-9]\)".*/\1/'
   register: kubeadm_version_major
+  check_mode: false
   changed_when: False
 
 - name: Retreive kubeadm Minor version
-  shell: set -o pipefail && kubeadm version | sed -e 's/.* Minor:"\([0-9]*\)".*/\1/'
+  ansible.builtin.shell: set -o pipefail && kubeadm version | sed -e 's/.* Minor:"\([0-9]*\)".*/\1/'
   register: kubeadm_version_minor
+  check_mode: false
   changed_when: False
 
 - name: Defined a default lb_kubemaster
-  set_fact:
+  ansible.builtin.set_fact:
-    lb_kubemaster: "{{ groups['KubernetesMasters'][0] }}"
+    lb_kubemaster: "{{ groups['KubernetesMasters_' ~ kubernetes_cluster_name][0] }}"
   when:
     - lb_kubemaster is undefined
 #    - groups['KubernetesMasters'] | length > 1
   changed_when: False
+  check_mode: false
 
+- name: Deploy First controler
+  block:
     - name: Deploy initial kubeadm config
-  template:
+      ansible.builtin.template:
         src: kubeadm-config.yaml.j2
         dest: /root/kubeadm-config.yaml
         owner: root
         group: root
         mode: 0600
-  when:
-    - groups['KubernetesMasterConfigured'] is not defined
-    - groups['KubernetesMasters'][0] == ansible_hostname
 
-- name: Init Kubernetes on {{ groups['KubernetesMasters'][0] }}
+    - name: Init Kubernetes on {{ groups['KubernetesMasters_' ~ kubernetes_cluster_name][0] }}
-  command: kubeadm init --config=/root/kubeadm-config.yaml
+      ansible.builtin.command: kubeadm init --config=/root/kubeadm-config.yaml
-  when:
-    - groups['KubernetesMasterConfigured'] is not defined
-    - groups['KubernetesMasters'][0] == ansible_hostname
 
     - name: Add {{ ansible_hostname }} to KubernetesMasterConfigured group
-  group_by:
+      ansible.builtin.group_by:
-    key: KubernetesMasterConfigured
+        key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }}
+      check_mode: false
+
   when:
-    - groups['KubernetesMasterConfigured'] is not defined
+    - groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined
-    - groups['KubernetesMasters'][0] == ansible_hostname
+    - groups['KubernetesMasters_' ~ kubernetes_cluster_name][0] == ansible_hostname
 
 # End of first controler
 
 - name: Test if server node already included
-  command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes {{ ansible_hostname | lower }}
+  ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes {{ ansible_hostname | lower }}
   delegate_to: "{{ lb_kubemaster }}"
   register: server_enrolled
   changed_when: False
   ignore_errors: yes
-  when:
+  check_mode: false
-    - groups['KubernetesMasterConfigured'] is not defined
 
 # - name: Deploy kubeadm config
-#  template:
+#   ansible.builtin.template:
 #     src: kubeadm-config.yaml.j2
 #     dest: /root/kubeadm-config.yaml
 #     owner: root
@@ -236,24 +308,24 @@
 #     - server_enrolled.rc == 1
 
 - name: Retreive certificats key on {{ lb_kubemaster }}
-  shell: set -o pipefail && kubeadm init phase upload-certs --upload-certs | grep -v upload-certs
+  ansible.builtin.shell: set -o pipefail && kubeadm init phase upload-certs --upload-certs | grep -v upload-certs
   register: kubernetes_certificateKey
+  check_mode: false
   delegate_to: "{{ lb_kubemaster }}"
   when:
-    - groups['KubernetesMasterConfigured'] is not defined
     - server_enrolled.rc == 1
     - kubernetes_master|bool
 
 - name: Retreive token on "{{ lb_kubemaster }}"
-  command: kubeadm token create
+  ansible.builtin.command: kubeadm token create
   register: kubetoken
   delegate_to: "{{ lb_kubemaster }}"
+  check_mode: false
   when:
-    - groups['KubernetesMasterConfigured'] is not defined
     - server_enrolled.rc == 1
 
 - name: Retreive hash certificat
-  shell: >
+  ansible.builtin.shell: >
     set -o pipefail &&
     openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt |
     openssl rsa -pubin -outform der 2>/dev/null |
@@ -261,23 +333,21 @@
     sed 's/^.* //'
   register: cacerthash
   delegate_to: "{{ lb_kubemaster }}"
+  check_mode: false
   when:
-    - groups['KubernetesMasterConfigured'] is not defined
     - server_enrolled.rc == 1
 
 - name: Deploy kubeadm config
-  template:
+  ansible.builtin.template:
     src: kubeadm-config.yaml.j2
     dest: /root/kubeadm-config.yaml
     owner: root
     group: root
     mode: 0600
   when:
-    - groups['KubernetesMasterConfigured'] is not defined
     - server_enrolled.rc == 1
 
 - name: Join '{{ ansible_hostname }}' to Kubernetes cluster
-  command: kubeadm join --config=/root/kubeadm-config.yaml
+  ansible.builtin.command: kubeadm join --config=/root/kubeadm-config.yaml
   when:
-    - groups['KubernetesMasterConfigured'] is not defined
     - server_enrolled.rc == 1

tasks/install_server.yml

@@ -1,52 +1,53 @@
 ---
+- name: Include vars for not taint Kubernetes masters
+  ansible.builtin.include_vars: masters.yml
+  when:
+    - kubernetes_master|bool
+    - not kubernetes_master_taint|bool
+
+- name: Add master to KubernetesMasters_ClusterName group
+  ansible.builtin.group_by:
+    key: KubernetesMasters_{{ kubernetes_cluster_name }}
+  check_mode: false
+  when:
+    - "'KubernetesMasters' in group_names"
+
+- name: Add node to KubernetesNodes_ClusterName group
+  ansible.builtin.group_by:
+    key: KubernetesNodes_{{ kubernetes_cluster_name }}
+  check_mode: false
+  when:
+    - "'KubernetesNodes' in group_names"
+
+
 - name: Disable SWAP since kubernetes can't work with swap enabled (1/2)
-  command: swapoff -a
+  ansible.builtin.command: swapoff -a
   changed_when: false
 
 - name: Remove swapfile from /etc/fstab (2/2)
-  mount:
+  ansible.posix.mount:
     name: swap
     fstype: swap
     state: absent
 
 - name: Create a thin pool for kubernetes
-  lvol:
+  community.general.lvol:
     vg: vg_sys
     thinpool: kubernetes
-    size: 20g
+    size: "{{ lv_kubernetes_size | default('20g') }}"
 
-# Install API loadbalancer
+## Install API loadbalancer
-- include_tasks: "load_balancer.yml"
+# - ansible.builtin.include_tasks: "load_balancer.yml"
-  when:
+#   when:
-    - kubernetes_master|bool
+#     - kubernetes_master|bool
-    - groups['KubernetesMasters'] | length > 1
+#     - groups['KubernetesMasters'] | length > 1
-
-- name: Audit policies directory
-  file:
-    path: "/etc/kubernetes/policies"
-    state: directory
-    owner: root
-    group: root
-    mode: 0700
-  when:
-    - kubernetes_master|bool
-
-- name: Configure audit policy
-  copy:
-    src: "etc/kubernetes/policies/audit-policy.yaml"
-    dest: "/etc/kubernetes/policies/audit-policy.yaml"
-    group: root
-    owner: root
-    mode: 0644
-  when:
-    - kubernetes_master|bool
 
 - name: Kubernetes cluster with kubeadm
-  include_tasks: "cluster_kubeadm.yml"
+  ansible.builtin.include_tasks: "cluster_kubeadm.yml"
   when:
     - kubernetes_cri != "k3s"
 - name: Kubernetes cluster with k3s
-  include_tasks: "cluster_k3s.yml"
+  ansible.builtin.include_tasks: "cluster_k3s.yml"
   when:
     - kubernetes_cri == "k3s"
 
@@ -55,7 +56,7 @@
 #
 
 - name: Make /root/.kube directory
-  file:
+  ansible.builtin.file:
     path: "/root/.kube"
     owner: root
     group: root
@@ -65,10 +66,10 @@
     - kubernetes_master|bool
 
 - name: Copy kubeconfig file from /etc/kubernetes/admin.conf
-  copy:
+  ansible.builtin.copy:
     src: "/etc/kubernetes/admin.conf"
     dest: /root/.kube/config
-    remote_src: yes
+    remote_src: true
     owner: root
     group: root
     mode: 0600
@@ -77,10 +78,10 @@
     - kubernetes_cri != "k3s"
 
 - name: Copy kubeconfig file from /etc/rancher/k3s/k3s.yaml
-  copy:
+  ansible.builtin.copy:
     src: "/etc/rancher/k3s/k3s.yaml"
     dest: /root/.kube/config
-    remote_src: yes
+    remote_src: true
     owner: root
     group: root
     mode: 0600
@@ -88,27 +89,40 @@
     - kubernetes_master|bool
     - kubernetes_cri == "k3s"
 
+- name: Make link from /etc/rancher/k3s/k3s.yaml to /etc/kubernetes/admin.conf
+  file:
+    src: "/etc/rancher/k3s/k3s.yaml"
+    state: link
+    dest: "/etc/kubernetes/admin.conf"
+    force: yes
+    owner: root
+    group: root
+  when:
+    - kubernetes_master|bool
+    - kubernetes_cri == "k3s"
+
 #
 # Manque autoconfig de .kube/config local
 #
 
 # - name: Fetching CA certificat 
-#  copy:
+#   ansible.builtin.copy:
 #     src: /etc/kubernetes/pki/ca.crt
 #     dest: /root/.kube/{{ kubernetes_cluster_name }}/ca.crt
 #   when:
 #     - kubernetes_master|bigip_pool
 
+# kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints --no-headers
 - name: Check if a node is still tainted
-  command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes '{{ ansible_hostname | lower }}' -o jsonpath='{.spec.taints}'
+  ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes '{{ ansible_hostname | lower }}' -o jsonpath='{.spec.taints}'
   register: current_taint
+  check_mode: false
   when:
     - kubernetes_master_taint|bool
 
-- name: taint the machine if needed
+- name: Taint the machine if needed
-#  command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master-
+#  ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master-
-  command: kubectl taint nodes '{{ ansible_hostname | lower }}' node-role.kubernetes.io/master-
+  ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes '{{ ansible_hostname | lower }}' node-role.kubernetes.io/master-
   when:
     - kubernetes_master_taint|bool
     - current_taint.stdout
-

tasks/load_balancer.yml

@@ -1,22 +1,21 @@
 ---
   - name: Install needed packages
-    package:
+    ansible.builtin.package:
-      name: "{{ item }}"
+      name:
-      state: present
-      update_cache: yes
-    with_items:
         - keepalived
         - curl
+      state: present
+      update_cache: true
     notify: Restart keepalived
   - name: Install check_apiserver.sh script for keepalived
-    template:
+    ansible.builtin.template:
       src: etc/keepalived/check_apiserver.sh.j2
       dest: /etc/keepalived/check_apiserver.sh
       owner: root
       group: root
       mode: 0755
   - name: Install keepalived config file
-    template:
+    ansible.builtin.template:
       src: etc/keepalived/keepalived.conf.j2
       dest: /etc/keepalived/keepalived.conf
       owner: root
@@ -28,7 +27,7 @@
       - groups['KubernetesMasters'][0] == ansible_hostname
     notify: Restart keepalived
   - name: Install keepalived config file
-    template:
+    ansible.builtin.template:
       src: etc/keepalived/keepalived.conf.j2
       dest: /etc/keepalived/keepalived.conf
       owner: root
@@ -39,3 +38,6 @@
     when:
       - not groups['KubernetesMasters'][0] == ansible_hostname
     notify: Restart keepalived
+
+  - name: Flush handlers
+    ansible.builtin.meta: flush_handlers

tasks/main.yml

@@ -1,55 +1,63 @@
 ---
+- name: Kubernetes Install
+  tags:
+    - kubernetes
+  block:
     - name: Include vars for {{ ansible_os_family }}
-  include_vars: "{{ ansible_os_family }}.yml"
+      ansible.builtin.include_vars: "{{ ansible_os_family }}.yml"
 
     - name: Define vars for master
-  set_fact:
+      ansible.builtin.set_fact:
         kubernetes_server: true
         kubernetes_master: true
         kubernetes_master_taint: false
+      check_mode: false
       when:
         - "'KubernetesMasters' in group_names"
         - "'KubernetesNodes' not in group_names"
 
     - name: Define vars for node
-  set_fact:
+      ansible.builtin.set_fact:
         kubernetes_server: true
         kubernetes_master: false
         kubernetes_master_taint: false
+      check_mode: false
       when:
         - "'KubernetesNodes' in group_names"
         - "'KubernetesMasters' not in group_names"
 
     - name: Define vars for taint master
-  set_fact:
+      ansible.builtin.set_fact:
         kubernetes_server: true
         kubernetes_master: true
         kubernetes_master_taint: true
+      check_mode: false
       when:
         - "'KubernetesNodes' in group_names"
         - "'KubernetesMasters' in group_names"
 
     - name: Define vars for tooling
-  set_fact:
+      ansible.builtin.set_fact:
         kubernetes_sever: false
+      check_mode: false
       when:
         - "'KubernetesMasters' not in group_names"
         - "'KubernetesNodes' not in group_names"
 
     - name: Install kubernetes rules for {{ ansible_os_family }} OS family
-  include_tasks: "{{ ansible_os_family }}.yml"
+      ansible.builtin.include_tasks: "{{ ansible_os_family }}.yml"
 
-- name: Install kubernetes tools
+  #   - name: Install kubernetes tools
-  package:
+  #     ansible.builtin.package:
-    name: "{{ kubernetes_package_name }}"
+  #       name: "{{ kubernetes_package_name }}"
-    state: present
+  #       state: present
-    update_cache: yes
+  #       update_cache: true
-#  notify: Restart kubelet
+  ##     notify: Restart kubelet
-  when:
+  #     when:
-    - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s")
+  #       - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s")
 
     - name: Include kubernetes server rules
-  include_tasks: "install_server.yml"
+      ansible.builtin.include_tasks: "install_server.yml"
       when:
         - kubernetes_server|bool
 

templates/etc/firewalld/services/kubernetes.xml.j2

@@ -9,6 +9,12 @@
 {% if kubernetes_master == true %}
 # Kubernetes API server, used by all
   <port protocol="tcp" port="6443"/>
+{% endif %}
+{% if kubernetes_cri == "k3s" %}
+# K3S with flannel and wireguard
+  <port protocol="udp" port="51820"/>
+{% else %}
+{% if kubernetes_master == true %}
 # etcd server client API, used by kube-apiserver and etcd
   <port protocol="tcp" port="2379"/>
   <port protocol="tcp" port="2380"/>
@@ -18,8 +24,8 @@
   <port protocol="tcp" port="10251"/>
 # kube-controler-manager, used by self
   <port protocol="tcp" port="10252"/>
-# ???
+# Read-only Kubelet API (Deprecated)
-  <port protocol="tcp" port="10255"/>
+#  <port protocol="tcp" port="10255"/>
 {% else %}
   <port protocol="tcp" port="10250"/>
 {% endif %}
@@ -35,4 +41,5 @@
   <port protocol="udp" port="6783-6784"/>
   <port protocol="tcp" port="6783"/>
 {% endif %}
+{% endif %}
 </service>

templates/etc/kubernetes/audit-webhook-kubeconfig.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: falco
+  cluster:
+    server: http://$FALCO_SERVICE_CLUSTERIP:8765/k8s-audit
+contexts:
+- context:
+    cluster: falco
+    user: ""
+  name: default-context
+current-context: default-context
+preferences: {}
+users: []

templates/etc/rancher/k3s/config.yaml.j2

@@ -0,0 +1,55 @@
+{% if 'Vpn' in group_names %}
+flannel-backend: vxlan
+{% else %}
+flannel-backend: wireguard-native
+{% endif %}
+protect-kernel-defaults: true
+{% if kubernetes_master|bool %}
+secrets-encryption: true
+kube-apiserver-arg:
+  - "enable-admission-plugins=NodeRestriction,AlwaysPullImages,EventRateLimit"
+  - 'admission-control-config-file=/etc/kubernetes/psa.yaml'
+  - 'audit-log-path=/var/log/apiserver/audit.log'
+  - 'audit-policy-file=/etc/kubernetes/policies/audit-policy.yaml'
+  - 'audit-log-maxage=30'
+  - 'audit-log-maxbackup=10'
+  - 'audit-log-maxsize=100'
+#  - "request-timeout=300s"
+kube-controller-manager-arg:
+  - 'terminated-pod-gc-threshold=10'
+cluster-cidr:
+  - {{ kubernetes_pods_network }}
+service-cidr:
+  - {{ kubernetes_svc_network }}
+{% if vars['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined %}
+cluster-init: true
+{% else %}
+server: https://{{ kubernetes_master }}:6443
+token: ${NODE_TOKEN}
+{% endif %}
+{% else %}
+server: https://{{ kubernetes_master }}:6443
+token: ${NODE_TOKEN}
+{% endif %}
+kubelet-arg:
+  - 'streaming-connection-idle-timeout=5m'
+  - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
+{% if false %}
+  - 'feature-gates=NodeSwap=true,CloudDualStackNodeIPs=true'
+{% endif %}
+{% if ansible_os_family == "RedHat" %}
+selinux: true
+{% endif %}
+#embedded-registry: true
+disable:
+  - traefik
+{% if lookup('vars', 'ansible_' + kubernetes_interface ) != ansible_host %}
+node-external-ip: {{ ansible_host }}
+{% endif %}
+node-ip: {{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}
+{% if false %}
+# node-external-ip: 1.2.3.4
+#node-label:
+#  - "foo=bar"
+#  - "something=amazing"
+{% endif %}

templates/etc/sysconfig/kubelet.j2

@@ -1,2 +1,2 @@
 #https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates
-KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_default_ipv4.address }}"
+KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}"

templates/etc/systemd/system/k3s.service.j2

@@ -7,9 +7,9 @@ After=network-online.target
 Type=notify
 EnvironmentFile=/etc/systemd/system/k3s.service.env
 {% if kubernetes_master|bool %}
-ExecStart=/usr/local/bin/k3s server --flannel-backend=wireguard --disable traefik --secrets-encryption
+ExecStart=/usr/local/bin/k3s server
 {% else %}
-ExecStart=/usr/local/bin/k3s agent --server https://{{ kubernetes_master }}:6443 --token ${NODE_TOKEN}
+ExecStart=/usr/local/bin/k3s agent
 {% endif %}
 KillMode=process
 Delegate=yes

templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2

@@ -1,2 +1,2 @@
 [Service]
-Environment=KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_default_ipv4.address }}"
+Environment=KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}"

templates/etc/systemd/system/kubelet.service.d/11-cgroups.conf.j2

@@ -0,0 +1,5 @@
+# https://stackoverflow.com/a/57456786
+# https://stackoverflow.com/questions/57456667/failed-to-get-kubelets-cgroup
+[Service]
+CPUAccounting=true
+MemoryAccounting=true

templates/etc/systemd/system/kubelet.service.d/20-allow-swap.conf.j2

@@ -0,0 +1 @@
+[Service] Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false"

templates/etc/yum.repos.d/kubernetes.repo.j2

@@ -1,7 +1,7 @@
 [kubernetes]
 name=Kubernetes
 baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-{{ ansible_machine }}
-enabled=1
+enabled=0
 gpgcheck=1
 repo_gpgcheck=1
 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg

templates/kubeadm-config.yaml.j2

@@ -1,4 +1,4 @@
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
 {% if kubetoken is defined %}
 bootstrapTokens:
@@ -14,8 +14,9 @@ nodeRegistration:
 {% elif  kubernetes_cri == "docker" %}
   criSocket: "/var/run/docker.sock"
 {% endif %}
+  name: {{ ansible_hostname }}
 {% if false %}
-  name: "ec2-10-100-0-1"
+  imagePullPolicy: IfNotPresent
   taints:
   - key: "kubeadmNode"
     value: "master"
@@ -25,70 +26,47 @@ nodeRegistration:
 {% if ansible_service_mgr == "systemd" %}
     cgroup-driver: "systemd"
 {% endif %}
-    container-runtime: "remote"
     runtime-request-timeout: "5m"
-{% if kubernetes_cri == "containerd" %}
+    node-ip: {{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}
-    container-runtime-endpoint: "unix:///run/containerd/containerd.sock"
-{% elif kubernetes_cri == "cri-o" %}
-    container-runtime-endpoint: "unix:///var/run/crio/crio.sock"
-{% endif %}
-    node-ip: {{ ansible_default_ipv4.address }}
     read-only-port: "10255"
   ignorePreflightErrors:
   - SystemVerification
+{% if (kubernetes_master|bool and not kubernetes_master_taint|bool) %}
+  - NumCPU
+{% endif %}
 {% if true == false %}
   - IsPrivilegedUser
 {% endif %}
 localAPIEndpoint:
-  advertiseAddress: "{{ ansible_default_ipv4.address }}"
+  advertiseAddress: "{{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}"
   bindPort: 6443
-{% if kubernetes_master|bool and groups['KubernetesMasterConfigured'] is defined %}
+{% if kubernetes_master|bool and groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %}
 certificateKey: "{{ kubernetes_certificateKey.stdout }}"
 {% endif %}
+{% if kubernetes_master|bool and groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined %}
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/v1beta3
-kind: JoinConfiguration
-{% if kubernetes_master|bool %}
-controlPlane:
-  localAPIEndpoint:
-    advertiseAddress: "{{ ansible_default_ipv4.address }}"
-    bindPort: 6443
-{% if groups['KubernetesMasterConfigured'] is defined %}
-  certificateKey: "{{ kubernetes_certificateKey.stdout }}"
-{% endif %}
-{% endif %}
-discovery:
-  bootstrapToken:
-    apiServerEndpoint: "{{ lb_kubemaster }}:6443"
-{% if groups['KubernetesMasterConfigured'] is defined %}
-    caCertHashes:
-    - sha256:{{ cacerthash.stdout }} 
-    token: "{{ kubetoken.stdout }}"
-{% endif %}
-nodeRegistration:
-  kubeletExtraArgs:
-    node-ip: {{ ansible_default_ipv4.address }}
-    read-only-port: "10255"
-  ignorePreflightErrors:
-  - SystemVerification
----
-apiVersion: kubeadm.k8s.io/v1beta2
 kind: ClusterConfiguration
 kubernetesVersion: stable
 {% if lbip_kubeapiserver is defined %}
 controlPlaneEndpoint: "{{ lbip_kubeapiserver }}:6443"
 {% else %}
-controlPlaneEndpoint: "{{ ansible_default_ipv4.address }}:6443"
+controlPlaneEndpoint: "{{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}:6443"
 {% endif %}
 apiServer:
   extraArgs:
-    enable-admission-plugins: NodeRestriction,PodSecurityPolicy
+    enable-admission-plugins: NodeRestriction,AlwaysPullImages,EventRateLimit
     authorization-mode: "Node,RBAC"
     audit-policy-file: "/etc/kubernetes/policies/audit-policy.yaml"
     audit-log-path: "/var/log/apiserver/audit.log"
     audit-log-maxage: "30"
     audit-log-maxbackup: "10"
     audit-log-maxsize: "100"
+{% if false %}
+# Falco
+    audit-webhook-config-file: "/etc/kubernetes/policies/audit-webhook-kubeconfig"
+    audit-webhook-batch-max-wait: "5s"
+{% endif %}
   extraVolumes:
   - name: "audit-log"
     hostPath: "/var/log/apiserver"
@@ -96,35 +74,163 @@ apiServer:
     readOnly: false
     pathType: DirectoryOrCreate
   - name: "audit-policies"
-    hostPath: "/etc/kubernetes/policies"
+    hostPath: "/etc/kubernetes/policies/audit-policy.yaml"
-    mountPath: "/etc/kubernetes/policies"
+    mountPath: "/etc/kubernetes/policies/audit-policy.yaml"
     readOnly: false
-    pathType: DirectoryOrCreate
+    pathType: File
 {% if lb_kubemaster is defined %}
   certSANs:
   - "{{ lb_kubemaster }}"
 {% endif %}
 {% if kubernetes_network == "flannel" or kubernetes_network == "calico" %}
 networking:
-{% if kubernetes_network == "flannel" %}
+  podSubnet: "{{ kubernetes_pods_network }}"
-  podSubnet: "10.244.0.0/16"
+  serviceSubnet: "{{ kubernetes_svc_network }}"
-{% elif kubernetes_network == "calico" %}
+{% endif %}
-  podSubnet: "192.168.0.0/16"
+controllerManager:
+  extraArgs:
+    bind-address: 0.0.0.0
+scheduler:
+  extraArgs:
+    bind-address: 0.0.0.0
+etcd:
+  local:
+    dataDir: /var/lib/etcd
+    extraArgs: 
+        listen-metrics-urls: http://0.0.0.0:2381
+{% endif %}
+{% if not kubernetes_master|bool or kubernetes_master|bool and groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %}
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: JoinConfiguration
+{% if kubernetes_master|bool %}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: "{{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}"
+    bindPort: 6443
+{% if groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %}
+  certificateKey: "{{ kubernetes_certificateKey.stdout }}"
+{% endif %}
+{% endif %}
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: "{{ lb_kubemaster }}:6443"
+{% if groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %}
+    caCertHashes:
+    - sha256:{{ cacerthash.stdout }} 
+    token: "{{ kubetoken.stdout }}"
+{% endif %}
+nodeRegistration:
+  kubeletExtraArgs:
+    node-ip: {{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}
+#    read-only-port: "10255"
+  ignorePreflightErrors:
+  - SystemVerification
+{% if kubernetes_swap_enabled is defined and kubernetes_swap_enabled|bool %}
+  - Swap
+{% endif %}
+{% if (kubernetes_master|bool and not kubernetes_master_taint|bool) %}
+  - NumCPU
 {% endif %}
 {% endif %}
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
+metricsBindAddress: "0.0.0.0:10249"
 {% if kubernetes_kubeproxy_mode is defined %}
 mode: {{ kubernetes_kubeproxy_mode }}
+{% if kubernetes_kubeproxy_mode == "ipvs" %}
+ipvs:
+  strictARP: true
+{% endif %}
 {% endif %}
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
-runtimeRequestTimeout: 5m
+#authentication:
+#  anonymous:
+#    enabled: false
+#  webhook:
+#    cacheTTL: 2m0s
+#    enabled: true
+#  x509:
+#    clientCAFile: /etc/kubernetes/pki/ca.crt
+#authorization:
+#  mode: Webhook
+#  webhook:
+#    cacheAuthorizedTTL: 5m0s
+#    cacheUnauthorizedTTL: 30s
 {% if ansible_service_mgr == "systemd" %}
 cgroupDriver: systemd
 {% endif %}
+#cgroupsPerQOS: true
+#clusterDNS:
+#- 10.96.0.10
+#clusterDomain: cluster.local
+#configMapAndSecretChangeDetectionStrategy: Watch
+#containerLogMaxFiles: 5
+#containerLogMaxSize: 10Mi
+#contentType: application/vnd.kubernetes.protobuf
+#cpuCFSQuota: true
+#cpuCFSQuotaPeriod: 100ms
+#cpuManagerPolicy: none
+#cpuManagerReconcilePeriod: 10s
+#enableControllerAttachDetach: true
+#enableDebuggingHandlers: true
+#enforceNodeAllocatable:
+#- pods
+#eventBurst: 10
+#eventRecordQPS: 5
+#evictionHard:
+#  imagefs.available: 15%
+#  memory.available: 500Mi
+#  nodefs.available: 10%
+#  nodefs.inodesFree: 5%
+#evictionPressureTransitionPeriod: 5m0s
+{% if kubernetes_swap_enabled is defined and kubernetes_swap_enabled|bool %}
+# Activation du swap
+failSwapOn: false
+featureGates:
+  NodeSwap: true
+memorySwap:
+  swapBehavior: UnlimitedSwap
+{% endif %}
+#fileCheckFrequency: 20s
+#hairpinMode: promiscuous-bridge
+#healthzBindAddress: 127.0.0.1
+#healthzPort: 10248
+#httpCheckFrequency: 20s
+#imageGCHighThresholdPercent: 85
+#imageGCLowThresholdPercent: 80
+#imageMinimumGCAge: 2m0s
+#iptablesDropBit: 15
+#iptablesMasqueradeBit: 14
+#kubeAPIBurst: 10
+#kubeAPIQPS: 5
+#logging: {}
+#makeIPTablesUtilChains: true
+#maxOpenFiles: 1000000
+#maxPods: 110
+#memorySwap: {}
+#nodeLeaseDurationSeconds: 40
+#nodeStatusReportFrequency: 1m0s
+#nodeStatusUpdateFrequency: 10s
+#oomScoreAdj: -999
+#podPidsLimit: -1
+#port: 10250
+#registryBurst: 10
+#registryPullQPS: 5
+#resolvConf: /etc/resolv.conf
+#rotateCertificates: true
+runtimeRequestTimeout: 5m
+#serializeImagePulls: true
+#shutdownGracePeriod: 0s
+#shutdownGracePeriodCriticalPods: 0s
+#staticPodPath: /etc/kubernetes/manifests
+#streamingConnectionIdleTimeout: 4h0m0s
+#syncFrequency: 1m0s
+#topologyManagerPolicy: none
+#volumeStatsAggPeriod: 1m0s
 
 {% if false %}
 readOnlyPort: 1

templates/var/lib/rancher/k3s/server/manifests/np-00-intra-namespace.yaml.j2

@@ -0,0 +1,12 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: intra-namespace
+  namespace: kube-system
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            name: kube-system

templates/var/lib/rancher/k3s/server/manifests/np-01-default-network-dns-policy.yaml.j2

@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-network-dns-policy
+  namespace: kube-system
+spec:
+  ingress:
+  - ports:
+    - port: 53
+      protocol: TCP
+    - port: 53
+      protocol: UDP
+  podSelector:
+    matchLabels:
+      k8s-app: kube-dns
+  policyTypes:
+  - Ingress

templates/var/lib/rancher/k3s/server/manifests/np-03-metrics-server-traefik.yaml.j2

@@ -0,0 +1,42 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-metrics-server
+  namespace: kube-system
+spec:
+  podSelector:
+    matchLabels:
+      k8s-app: metrics-server
+  ingress:
+  - {}
+  policyTypes:
+  - Ingress
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-svclbtraefik-ingress
+  namespace: kube-system
+spec:
+  podSelector: 
+    matchLabels:
+      svccontroller.k3s.cattle.io/svcname: traefik
+  ingress:
+  - {}
+  policyTypes:
+  - Ingress
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-traefik-v121-ingress
+  namespace: kube-system
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: traefik
+  ingress:
+  - {}
+  policyTypes:
+  - Ingress
+---

vars/RedHat.yml

@@ -1,8 +1,9 @@
 ---
 kubernetes_package_name: 
-  - kubectl
+  - kubectl-{{ kubernetes_version }}
-  - kubelet
+  - kubelet-{{ kubernetes_version }}
-  - kubeadm
+  - kubeadm-{{ kubernetes_version }}
   - iproute-tc
+  - ipvsadm
 #kubernetes_remove_packages_name:
 #  - kubernetes.io

vars/masters.yml

@@ -0,0 +1,3 @@
+---
+lv_containers_size: 2g
+lv_kubernetes_size: 8g