diff --git a/README.md b/README.md index abaa25b..2b6dc5f 100644 --- a/README.md +++ b/README.md @@ -9,5 +9,3 @@ Deploy kubernetes https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2?tab=doc https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/ - -https://github.com/XenitAB/spegel diff --git a/defaults/main.yml b/defaults/main.yml index 1a656c5..3bbf3a5 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -2,16 +2,7 @@ # value for kubernetes_cri: containerd, cri-o #kubernetes_cri: "containerd" kubernetes_server: false -kubernetes_interface: '{{ ansible_default_ipv4.interface }}' # value for kuberntes_network: flannel, calico, weave-net #kubernetes_network: weave-net kubernetes_kubeproxy_mode: ipvs -kubernetes_version: 1.31.5 -kubernetes_k3s_version: 1.31.5+k3s1 -#kubernetes_pods_network: "10.244.0.0/16" -#kubernetes_svc_network: "10.96.0.0/12" -kubernetes_pods_network: "10.42.0.0/16" -kubernetes_svc_network: "10.43.0.0/16" -lb_auth_pass: 1be344d62acc46c6858ae8475668a245 -kubernetes_swap_enabled: false -kubernetes_lvm: true +kubernetes_version: 1.20.1 diff --git a/files/etc/NetworkManager/conf.d/calico.conf b/files/etc/NetworkManager/conf.d/calico.conf deleted file mode 100644 index b4ac62a..0000000 --- a/files/etc/NetworkManager/conf.d/calico.conf +++ /dev/null @@ -1,3 +0,0 @@ -# https://docs.tigera.io/calico/latest/operations/troubleshoot/troubleshooting#configure-networkmanager -[keyfile] -unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:wireguard.cali diff --git a/files/etc/kubernetes/policies/audit-policy.yaml b/files/etc/kubernetes/policies/audit-policy.yaml index 9067920..25b8fd0 100644 --- a/files/etc/kubernetes/policies/audit-policy.yaml +++ b/files/etc/kubernetes/policies/audit-policy.yaml @@ -10,15 +10,7 @@ rules: - group: "" # Resource "pods" doesn't match requests to any subresource of pods, # which is consistent with the RBAC policy. - resources: ["pods", "deployments"] - - - level: RequestResponse - resources: - - group: "rbac.authorization.k8s.io" - # Resource "pods" doesn't match requests to any subresource of pods, - # which is consistent with the RBAC policy. - resources: ["clusterroles", "clusterrolebindings"] - + resources: ["pods"] # Log "pods/log", "pods/status" at Metadata level - level: Metadata resources: @@ -56,17 +48,11 @@ rules: # The empty string "" can be used to select non-namespaced resources. namespaces: ["kube-system"] - # Log configmap changes in all other namespaces at the RequestResponse level. - - level: RequestResponse - resources: - - group: "" # core API group - resources: ["configmaps"] - - # Log secret changes in all other namespaces at the Metadata level. + # Log configmap and secret changes in all other namespaces at the Metadata level. - level: Metadata resources: - group: "" # core API group - resources: ["secrets"] + resources: ["secrets", "configmaps"] # Log all other resources in core and extensions at the Request level. - level: Request diff --git a/files/etc/kubernetes/psa.yaml b/files/etc/kubernetes/psa.yaml deleted file mode 100644 index fe13d52..0000000 --- a/files/etc/kubernetes/psa.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: apiserver.config.k8s.io/v1 -kind: AdmissionConfiguration -plugins: -- name: PodSecurity - configuration: - apiVersion: pod-security.admission.config.k8s.io/v1beta1 - kind: PodSecurityConfiguration - defaults: - enforce: "restricted" - enforce-version: "latest" - audit: "restricted" - audit-version: "latest" - warn: "restricted" - warn-version: "latest" - exemptions: - usernames: [] - runtimeClasses: [] - namespaces: [kube-system, cis-operator-system] -- name: EventRateLimit - configuration: - apiVersion: eventratelimit.admission.k8s.io/v1alpha1 - kind: Configuration - limits: - - burst: 20000 - qps: 5000 - type: Server - path: "" diff --git a/meta/main.yml b/meta/main.yml index ad6ee72..5968623 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -6,13 +6,11 @@ galaxy_info: galaxy_tags: [] license: GPL2 platforms: - - name: CentOS - version: - - 7 - - 8 - - 9 - - name: RedHat - version: - - 7 - - 8 - - 9 + - name: CentOS + version: + - 7 + - 8 + - name: RedHat + version: + - 7 + - 8 diff --git a/tasks/Debian.yml b/tasks/Debian.yml index c4b91da..9952f0a 100644 --- a/tasks/Debian.yml +++ b/tasks/Debian.yml @@ -1,21 +1,21 @@ --- -- name: Add docker apt key - ansible.builtin.apt_key: +- name: add docker apt key + apt_key: url: https://download.docker.com/linux/ubuntu/gpg state: present when: - docker_ver == "docker_ce" -- name: Add docker repository - ansible.builtin.apt_repository: +- name: add docker repository + apt_repository: repo: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable' state: present - update_cache: true + update_cache: yes when: - docker_ver == "docker_ce" - name: "Ensure GRUB_CMDLINE_LINUX is updated" - ansible.builtin.lineinfile: + lineinfile: dest: /etc/default/grub regexp: '^(GRUB_CMDLINE_LINUX=".*)"$' line: '\1 cgroup_enable=memory swapaccount=1"' @@ -24,12 +24,12 @@ - not docker_installed.stat.exists - name: "Update grub.conf" - ansible.builtin.command: update-grub + command: update-grub when: - not docker_installed.stat.exists - name: "Ensure DEFAULT_FORWARD_POLICY in /etc/default/ufw is updated" - ansible.builtin.lineinfile: + lineinfile: dest: /etc/default/ufw regexp: '^(DEFAULT_FORWARD_POLICY=").*"$' line: '\1ACCEPT"' @@ -38,11 +38,11 @@ tags: [docker,firewall] # Need Certificat ? Only in local -# - name: "Add docker port 2376/TCP " -# ufw: rule=allow port=2376 proto=tcp -# notify: reload ufw -# tags: [docker,firewall] +#- name: "Add docker port 2376/TCP " +# ufw: rule=allow port=2376 proto=tcp +# notify: reload ufw +# tags: [docker,firewall] -# - name: "Start UFW rules" -# service: name=ufw state=started -# tags: [docker,firewall] +#- name: "Start UFW rules" +# service: name=ufw state=started +# tags: [docker,firewall] diff --git a/tasks/RedHat.yml b/tasks/RedHat.yml index bcbf718..7eaf639 100644 --- a/tasks/RedHat.yml +++ b/tasks/RedHat.yml @@ -1,159 +1,59 @@ --- -# - name: Add kubernetes repository -# yumrepo: -# name: kubernetes -# description: "Kubernetes Repository" -# baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-$releasever-x86_64 -# gpgcheck: yes -# enabled: true -# gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg -# state: present +#- name: Add kubernetes repository +# yumrepo: +# name: kubernetes +# description: "Kubernetes Repository" +# baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-$releasever-x86_64 +# gpgcheck: yes +# enabled: yes +# gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg +# state: present -# - name: Add Official kubernetes's repo -# ansible.builtin.template: -# src: "etc/yum.repos.d/kubernetes.repo.j2" -# dest: "/etc/yum.repos.d/kubernetes.repo" -# group: root -# owner: root -# mode: 0644 -# when: -# - not ansible_machine == "armv7l" -# - not ansible_machine == "armv6l" -# - kubernetes_cri != "k3s" - -- name: Add Official kubernetes's repo on servers - ansible.builtin.yum_repository: - name: kubernetes - description: Kubernetes - baseurl: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/" - enabled: true - gpgcheck: true - repo_gpgcheck: true - gpgkey: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/repodata/repomd.xml.key" - exclude: kubelet kubeadm - become: true +- name: Add Official kubernetes's repo + template: + src: "etc/yum.repos.d/kubernetes.repo.j2" + dest: "/etc/yum.repos.d/kubernetes.repo" + group: root + owner: root + mode: 0644 when: - not ansible_machine == "armv7l" - not ansible_machine == "armv6l" - - kubernetes_server|bool - kubernetes_cri != "k3s" -- name: Add Official kubernetes's repo for Desktop - ansible.builtin.yum_repository: - name: kubernetes - description: Kubernetes - baseurl: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/" - enabled: true - gpgcheck: true - repo_gpgcheck: true - gpgkey: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version | regex_replace('^([0-9])\\.([0-9]*).*', '\\1.\\2') }}/rpm/repodata/repomd.xml.key" - exclude: kubelet kubeadm kubectl - become: true - when: - - not ansible_machine == "armv7l" - - not ansible_machine == "armv6l" - - not kubernetes_server|bool - -# - name: Redhat | Installing K8s Packages -# ansible.builtin.package: -# name: -# - kubectl -# - kubelet -# - kubeadm -# - iproute-tc -# - ipvsadm -# state: present -# disable_excludes: kubernetes -# become: true -# register: result -# until: result is successful - -# - name: Register kubernetes firewalld service -# ansible.builtin.template: -# src: "etc/firewalld/services/kubernetes.xml.j2" -# dest: "/etc/firewalld/services/kubernetes.xml" -# group: root -# owner: root -# mode: 0644 -# register: need_firewalld_reload -# when: -# - kubernetes_server|bool -# -# - name: Reload firewalld configuration -# ansible.builtin.service: -# name: firewalld -# state: reloaded -# enabled: true -# when: -# - kubernetes_server|bool -# - need_firewalld_reload is changed -# -## Définir interface -# - name: Open Firewalld -# ansible.posix.firewalld: -# zone: external -# service: kubernetes -# permanent: true -# state: enabled -# immediate: true -# when: -## - need_firewall|bool -## - firewall_name == "firewalld" -# - kubernetes_server|bool - -# - name: Create kubernetes firewalld zone -# ansible.posix.firewalld: -# zone: kubernetes -# permanent: true -# state: present -# when: -# - kubernetes_server|bool -- name: Add kubernetes networks to trusted firewalld zone - ansible.posix.firewalld: -# zone: kubernetes - zone: trusted - permanent: true - state: enabled - source: "{{ item }}" +- name: Register kubernetes firewalld service + template: + src: "etc/firewalld/services/kubernetes.xml.j2" + dest: "/etc/firewalld/services/kubernetes.xml" + group: root + owner: root + mode: 0644 + register: need_firewalld_reload when: - kubernetes_server|bool - with_items: - - "{{ kubernetes_pods_network }}" - - "{{ kubernetes_svc_network }}" -- name: Add kubernetes networks to trusted firewalld zone - ansible.posix.firewalld: -# zone: kubernetes - zone: trusted +#- name: Reload firewalld configuration +# service: +# name: firewalld +# state: reloaded +# enabled: yes +# when: +# - kubernetes_server|bool + +- name: reload firewalld to refresh service list + command: firewall-cmd --reload + when: + - need_firewalld_reload is changed + - kubernetes_server|bool + +# Définir interface +- name: Open Firewalld + firewalld: + service: kubernetes permanent: true state: enabled - source: "{{ item }}" + immediate: true when: +# - need_firewall|bool +# - firewall_name == "firewalld" - kubernetes_server|bool - - kubernetes_interface is defined -# - false - with_items: - - "{{ (lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.network + '/' + lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.netmask) | ansible.utils.ipaddr('net') }}" - -- name: Install kubernetes tools - ansible.builtin.dnf: - name: "{{ kubernetes_package_name }}" - enablerepo: "kubernetes" - state: present - update_cache: true - disable_excludes: kubernetes -# notify: Restart kubelet - when: - - ansible_pkg_mgr == "dnf" - - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s") - -- name: Install kubernetes tools - ansible.builtin.yum: - name: "{{ kubernetes_package_name }}" - enablerepo: "kubernetes" - state: present - update_cache: true -# notify: Restart kubelet - when: - - ansible_pkg_mgr == "yum" - - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s") diff --git a/tasks/cluster_k3s.yml b/tasks/cluster_k3s.yml index afe4024..9727eb0 100644 --- a/tasks/cluster_k3s.yml +++ b/tasks/cluster_k3s.yml @@ -1,46 +1,19 @@ --- - name: Install Wireguard - ansible.builtin.include_role: + include_role: name: wireguard - when: +# when: # - kubernetes_cni == "wireguard" - - "'Vpn' not in group_names" - -- name: Import Rancher key - ansible.builtin.rpm_key: - state: present - key: https://rpm.rancher.io/public.key - when: - - ansible_os_family == "RedHat" - -- name: Install the k3s-selinux rpm from a remote repo for yum distro - ansible.builtin.yum: - name: "https://github.com/k3s-io/k3s-selinux/releases/download/v1.6.stable.1/k3s-selinux-1.6-1.el7.noarch.rpm" - state: present - when: - - ansible_pkg_mgr == "yum" - - ansible_os_family == "RedHat" - - ansible_distribution_major_version == '7' - -- name: Install the k3s-selinux rpm from a remote repo for dnf distro - ansible.builtin.dnf: - name: "https://github.com/k3s-io/k3s-selinux/releases/download/v1.6.stable.1/k3s-selinux-1.6-1.el{{ ansible_distribution_major_version }}.noarch.rpm" - state: present - when: - - ansible_pkg_mgr == "dnf" - - ansible_os_family == "RedHat" - - ansible_distribution_major_version >= '8' - name: Check if /usr/local/bin/k3s already existe - ansible.builtin.stat: + stat: path: /usr/local/bin/k3s register: k3s_bin - check_mode: false changed_when: False -- name: Retreive k3s binary for x86_64 - ansible.builtin.get_url: - url: "https://github.com/rancher/k3s/releases/download/v{{ kubernetes_k3s_version | urlencode }}/k3s" +- name: retreive k3s binary for x86_64 + get_url: + url: "https://github.com/rancher/k3s/releases/download/v1.20.2%2Bk3s1/k3s" dest: "/usr/local/bin/k3s" group: root owner: root @@ -49,9 +22,9 @@ - not k3s_bin.stat.exists - ansible_machine == "x86_64" -- name: Retreive k3s binary for arm64 - ansible.builtin.get_url: - url: "https://github.com/rancher/k3s/releases/download/v{{ kubernetes_k3s_version | urlencode }}/k3s-arm64" +- name: retreive k3s binary for arm64 + get_url: + url: "https://github.com/rancher/k3s/releases/download/v1.20.2%2Bk3s1/k3s-arm64" dest: "/usr/local/bin/k3s" group: root owner: root @@ -60,9 +33,9 @@ - not k3s_bin.stat.exists - ansible_machine == "arm64" -- name: Retreive k3s binary for armv6/armv7 - ansible.builtin.get_url: - url: "https://github.com/rancher/k3s/releases/download/v{{ kubernetes_k3s_version | urlencode }}/k3s-armhf" +- name: retreive k3s binary for armv6/armv7 + get_url: + url: "https://github.com/rancher/k3s/releases/download/v1.20.2%2Bk3s1/k3s-armhf" dest: "/usr/local/bin/k3s" group: root owner: root @@ -72,7 +45,7 @@ - (ansible_machine == "armv7l") or (ansible_machine == "armv6l") - name: Create tools link - ansible.builtin.file: + file: src: "k3s" dest: "/usr/local/bin/{{ item }}" owner: root @@ -83,225 +56,49 @@ - "crictl" - "ctr" -- name: Create logical volume for k3s - when: - - kubernetes_lvm|bool - block: - - name: Create thin volumes for k3s - community.general.lvol: - vg: "{{ item.vg }}" - lv: "{{ item.name }}" - thinpool: kubernetes - size: "{{ item.size }}" - with_items: - - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"} - - - name: Create file system on containerd lv - community.general.filesystem: - fstype: ext4 - dev: "/dev/{{ item.vg }}/{{ item.name }}" - with_items: - - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"} - - - name: Mount logical volumes - ansible.posix.mount: - name: "{{ item.mount_point }}" - src: "/dev/{{ item.vg }}/{{ item.name }}" - fstype: ext4 - opts: "{{ item.mount_opts }}" - state: mounted - with_items: - - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"} - -- name: Ensure protect-kernel-defaults is set - ansible.posix.sysctl: - name: "{{ item.name }}" - value: "{{ item.value }}" - sysctl_file: /etc/sysctl.d/90-kubelet.conf - reload: true - with_items: - - { name: "vm.panic_on_oom", value: "0" } - - { name: "vm.overcommit_memory", value: "1" } - - { name: "kernel.panic", value: "10" } - - { name: "kernel.panic_on_oops", value: "1" } - when: - - kubernetes_server|bool - -- name: /etc/kubernetes directory - ansible.builtin.file: - path: "/etc/kubernetes" - state: directory - owner: root - group: root - mode: 0755 - when: - - kubernetes_master|bool - -- name: Configure Pod Security - ansible.builtin.copy: - src: "etc/kubernetes/psa.yaml" - dest: "/etc/kubernetes/psa.yaml" - group: root - owner: root - mode: 0644 - when: - - kubernetes_master|bool - -- name: Audit policies directory - ansible.builtin.file: - path: "/etc/kubernetes/policies" - state: directory - owner: root - group: root - mode: 0700 - when: - - kubernetes_master|bool - -- name: Configure audit policy - ansible.builtin.copy: - src: "etc/kubernetes/policies/audit-policy.yaml" - dest: "/etc/kubernetes/policies/audit-policy.yaml" - group: root - owner: root - mode: 0644 - when: - - kubernetes_master|bool - -# Check controlers -- name: Check if /etc/rancher/k3s/k3s.yaml already existe - ansible.builtin.stat: - path: /etc/rancher/k3s/k3s.yaml - register: st - check_mode: false - changed_when: False - when: - - kubernetes_master|bool - -- name: Create KubernetesMasterConfigured group - ansible.builtin.group_by: - key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }} - check_mode: false - when: - - kubernetes_master|bool - - st.stat.exists - -# First controler -- name: Configure first controler -# run_once: true - block: - - name: Create k3s directories on master nodes - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: 0700 - with_items: - - "/etc/rancher" - - "/etc/rancher/k3s" - - "/etc/rancher/k3s/config.yaml.d" - - "/var/lib/rancher" - - "/var/lib/rancher/k3s" - - "/var/lib/rancher/k3s/server" - - "/var/lib/rancher/k3s/server/manifests" - when: - - kubernetes_master|bool - - - name: Create k3s directories on all nodes - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: 0700 - with_items: - - "/var/lib/rancher/k3s/storage" - -# semanage fcontext -a -t container_file_t "/var/lib/rancher/k3s/storage(/.*)?" - - name: Allow K3S local-path provisioner to create directories in /var/lib/rancher/k3s/storage - community.general.sefcontext: - target: '/var/lib/rancher/k3s/storage(/.*)?' - setype: container_file_t - state: present - - name: Apply new SELinux file context to filesystem - ansible.builtin.command: restorecon -R /var/lib/rancher/k3s/storage/ - - - name: Deploy Network Policies - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "/{{ item }}" - owner: root - group: root - mode: 0600 - with_items: - - "var/lib/rancher/k3s/server/manifests/np-00-intra-namespace.yaml" - - "var/lib/rancher/k3s/server/manifests/np-01-default-network-dns-policy.yaml" - - "var/lib/rancher/k3s/server/manifests/np-03-metrics-server-traefik.yaml" - when: - - kubernetes_master|bool - - - name: Deploy systemd service - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "/{{ item }}" - owner: root - group: root - mode: 0600 - with_items: - - "etc/systemd/system/k3s.service" - - "etc/systemd/system/k3s.service.env" - - "etc/rancher/k3s/config.yaml" - when: - - ansible_service_mgr == "systemd" - - - name: Reload systemd - ansible.builtin.systemd: - daemon_reload: true - - - name: Enable k3s on boot - ansible.builtin.service: - name: k3s - state: started - enabled: true - - - name: Wait for k3s.yaml - wait_for: - path: /etc/rancher/k3s/k3s.yaml - - - name: Wait for token - wait_for: - path: /var/lib/rancher/k3s/server/token - - - name: Add {{ ansible_hostname }} to KubernetesMasterConfigured group - ansible.builtin.group_by: - key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }} - check_mode: false - - when: - - kubernetes_master|bool - - vars['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined - -# chmod -R 600 /var/lib/rancher/k3s/server/tls/*.crt - # Manque kubernetes_server_token, kubernetes_master url -# - name: Deploy systemd service -# ansible.builtin.template: -# src: "etc/systemd/system/{{ item }}.j2" -# dest: "/etc/systemd/system/{{ item }}" -# owner: root -# group: root -# mode: 0600 -# with_items: -# - "k3s.service" -# - "k3s.service.env" -# when: -# - ansible_service_mgr == "systemd" +- name: Deploy systemd service + template: + src: "etc/systemd/system/{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: 0600 + with_items: + - "k3s.service" + - "k3s.service.env" + when: + - ansible_service_mgr == "systemd" +- name: Create thin volumes for k3s + lvol: + vg: "{{ item.vg }}" + lv: "{{ item.name }}" + thinpool: kubernetes + size: "{{ item.size }}" + with_items: + - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"} +- name: create file system on containerd lv + filesystem: + fstype: ext4 + dev: "/dev/{{ item.vg }}/{{ item.name }}" + with_items: + - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"} + +- name: mount logical volumes + mount: + name: "{{ item.mount_point }}" + src: "/dev/{{ item.vg }}/{{ item.name }}" + fstype: ext4 + opts: "{{ item.mount_opts }}" + state: mounted + with_items: + - { name: var_lib_k3s, vg: vg_sys, size: 10g, mount_point: /var/lib/rancher/k3s, mount_opts: "discard"} - name: Enable k3s on boot - ansible.builtin.service: + service: name: k3s state: started - enabled: true + enabled: yes diff --git a/tasks/cluster_kubeadm.yml b/tasks/cluster_kubeadm.yml index c11bf3a..c15325d 100644 --- a/tasks/cluster_kubeadm.yml +++ b/tasks/cluster_kubeadm.yml @@ -1,46 +1,27 @@ --- - name: Install Containerd - ansible.builtin.include_role: + include_role: name: containerd when: - kubernetes_cri == "containerd" - # register: kubernetes_cri_changed + #register: kubernetes_cri_changed - name: Install CRI-O - ansible.builtin.include_role: + include_role: name: cri-o when: - kubernetes_cri == "cri-o" - # register: kubernetes_cri_changed + #register: kubernetes_cri_changed -# - name: Restart kubelet after kubernetes cri installation -# ansible.builtin.service: -# name: kubelet -# status: restarted -# when: -# - kubernetes_cri_changed is changed - -- name: Configure NetworkManager for Calico - ansible.builtin.copy: - src: "etc/NetworkManager/conf.d/calico.conf" - dest: "/etc/NetworkManager/conf.d/calico.conf" - group: root - owner: root - mode: 0644 - when: - - kubernetes_network == "calico" - - ansible_os_family == "RedHat" - register: kubernetes_network_networkmanager_changed - -- name: Restart kubelet after kubernetes cri installation - ansible.builtin.service: - name: NetworkManager - status: reload - when: - - kubernetes_network_networkmanager_changed is changed +#- name: Restart kubelet after kubernetes cri installation +# service: +# name: kubelet +# status: restarted +# when: +# - kubernetes_cri_changed is changed - name: Configuring IPVS kernel module to be load on boot - ansible.builtin.template: + template: src: "etc/modules-load.d/ipvs.conf.j2" dest: "/etc/modules-load.d/ipvs.conf" group: root @@ -50,7 +31,7 @@ - kubernetes_kubeproxy_mode == "ipvs" - name: Load IPVS kernel module for EL7 - community.general.modprobe: + modprobe: name: "{{ item }}" state: present with_items: @@ -66,7 +47,7 @@ - ansible_distribution_major_version == '7' - name: Load IPVS kernel module for EL8 - community.general.modprobe: + modprobe: name: "{{ item }}" state: present with_items: @@ -78,10 +59,10 @@ when: - kubernetes_kubeproxy_mode == "ipvs" - ansible_os_family == "RedHat" - - ansible_distribution_major_version >= '8' + - ansible_distribution_major_version == '8' - name: Create thin volumes for kubernetes - community.general.lvol: + lvol: vg: "{{ item.vg }}" lv: "{{ item.name }}" thinpool: kubernetes @@ -93,8 +74,8 @@ when: - kubernetes_master|bool -- name: Create file system on containerd lv - community.general.filesystem: +- name: create file system on containerd lv + filesystem: fstype: ext4 dev: "/dev/{{ item.vg }}/{{ item.name }}" with_items: @@ -104,8 +85,8 @@ when: - kubernetes_master|bool -- name: Mount logical volumes - ansible.posix.mount: +- name: mount logical volumes + mount: name: "{{ item.mount_point }}" src: "/dev/{{ item.vg }}/{{ item.name }}" fstype: ext4 @@ -120,14 +101,14 @@ - kubernetes_master|bool - name: Ensuring /var/lib/etcd/lost+found Folder does not exists - ansible.builtin.file: + file: path: "/var/lib/etcd/lost+found" state: "absent" when: - partition_formated is changed - name: Secure etcd directory - ansible.builtin.file: + file: path: "/var/lib/etcd" state: directory owner: root @@ -137,7 +118,7 @@ - kubernetes_master|bool - name: Ensuring /etc/systemd/system/kubelet.service.d Folder Exists - ansible.builtin.file: + file: path: "/etc/systemd/system/kubelet.service.d" state: "directory" group: root @@ -147,7 +128,7 @@ - ansible_service_mgr == "systemd" - name: Configure kubelet service - ansible.builtin.template: + template: src: "etc/{{ item }}.j2" dest: "/etc/{{ item }}" group: root @@ -159,35 +140,8 @@ when: - ansible_service_mgr == "systemd" -- name: Configure kubelet service for CRI-O - ansible.builtin.template: - src: "etc/{{ item }}.j2" - dest: "/etc/{{ item }}" - group: root - owner: root - mode: 0644 - with_items: - - "systemd/system/kubelet.service.d/11-cgroups.conf" - when: - - ansible_service_mgr == "systemd" - - kubernetes_cri == "cri-o" - -- name: Enable Swap for kubelet service - ansible.builtin.template: - src: "etc/{{ item }}.j2" - dest: "/etc/{{ item }}" - group: root - owner: root - mode: 0644 - with_items: - - "systemd/system/kubelet.service.d/20-allow-swap.conf" - when: - - ansible_service_mgr == "systemd" - - kubernetes_swap_enabled is defined - - kubernetes_swap_enabled|bool - - name: Configure kubelet service - ansible.builtin.template: + template: src: "etc/{{ item }}.j2" dest: "/etc/{{ item }}" group: root @@ -199,133 +153,107 @@ - not ansible_service_mgr == "systemd" - name: Enable kubelet on boot - ansible.builtin.service: + service: name: kubelet state: started - enabled: true - -- name: Audit policies directory - ansible.builtin.file: - path: "/etc/kubernetes/policies" - state: directory - owner: root - group: root - mode: 0700 - when: - - kubernetes_master|bool - -# https://v1-17.docs.kubernetes.io/docs/tasks/debug-application-cluster/falco/ -# https://github.com/falcosecurity/falco/blob/master/rules/k8s_audit_rules.yaml -# Ou récupération de ces règles pour une utilisation avec falco - -- name: Configure audit policy - ansible.builtin.copy: - src: "etc/kubernetes/policies/audit-policy.yaml" - dest: "/etc/kubernetes/policies/audit-policy.yaml" - group: root - owner: root - mode: 0644 - when: - - kubernetes_master|bool + enabled: yes # First controler - name: Check if /etc/kubernetes/admin.conf already existe - ansible.builtin.stat: + stat: path: /etc/kubernetes/admin.conf register: st - check_mode: false changed_when: False - name: Create KubernetesMasterConfigured group - ansible.builtin.group_by: - key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }} - check_mode: false + group_by: + key: KubernetesMasterConfigured when: - st.stat.exists - name: Retreive kubeadm Major version - ansible.builtin.shell: set -o pipefail && kubeadm version | sed 's/.*{Major:"\([0-9]\)".*/\1/' + shell: set -o pipefail && kubeadm version | sed 's/.*{Major:"\([0-9]\)".*/\1/' register: kubeadm_version_major - check_mode: false changed_when: False - name: Retreive kubeadm Minor version - ansible.builtin.shell: set -o pipefail && kubeadm version | sed -e 's/.* Minor:"\([0-9]*\)".*/\1/' + shell: set -o pipefail && kubeadm version | sed -e 's/.* Minor:"\([0-9]*\)".*/\1/' register: kubeadm_version_minor - check_mode: false changed_when: False - name: Defined a default lb_kubemaster - ansible.builtin.set_fact: - lb_kubemaster: "{{ groups['KubernetesMasters_' ~ kubernetes_cluster_name][0] }}" + set_fact: + lb_kubemaster: "{{ groups['KubernetesMasters'][0] }}" when: - lb_kubemaster is undefined # - groups['KubernetesMasters'] | length > 1 changed_when: False - check_mode: false - -- name: Deploy First controler - block: - - name: Deploy initial kubeadm config - ansible.builtin.template: - src: kubeadm-config.yaml.j2 - dest: /root/kubeadm-config.yaml - owner: root - group: root - mode: 0600 - - - name: Init Kubernetes on {{ groups['KubernetesMasters_' ~ kubernetes_cluster_name][0] }} - ansible.builtin.command: kubeadm init --config=/root/kubeadm-config.yaml - - - name: Add {{ ansible_hostname }} to KubernetesMasterConfigured group - ansible.builtin.group_by: - key: KubernetesMasterConfigured_{{ kubernetes_cluster_name }} - check_mode: false +- name: Deploy initial kubeadm config + template: + src: kubeadm-config.yaml.j2 + dest: /root/kubeadm-config.yaml + owner: root + group: root + mode: 0600 when: - - groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined - - groups['KubernetesMasters_' ~ kubernetes_cluster_name][0] == ansible_hostname + - groups['KubernetesMasterConfigured'] is not defined + - groups['KubernetesMasters'][0] == ansible_hostname + +- name: Init Kubernetes on {{ groups['KubernetesMasters'][0] }} + command: kubeadm init --config=/root/kubeadm-config.yaml + when: + - groups['KubernetesMasterConfigured'] is not defined + - groups['KubernetesMasters'][0] == ansible_hostname + +- name: Add {{ ansible_hostname }} to KubernetesMasterConfigured group + group_by: + key: KubernetesMasterConfigured + when: + - groups['KubernetesMasterConfigured'] is not defined + - groups['KubernetesMasters'][0] == ansible_hostname # End of first controler - name: Test if server node already included - ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes {{ ansible_hostname | lower }} + command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes {{ ansible_hostname | lower }} delegate_to: "{{ lb_kubemaster }}" register: server_enrolled changed_when: False ignore_errors: yes - check_mode: false + when: + - groups['KubernetesMasterConfigured'] is not defined -# - name: Deploy kubeadm config -# ansible.builtin.template: -# src: kubeadm-config.yaml.j2 -# dest: /root/kubeadm-config.yaml -# owner: root -# group: root -# mode: 600 -# when: -# - not groups['KubernetesMasters'][0] == ansible_hostname -# - server_enrolled.rc == 1 +#- name: Deploy kubeadm config +# template: +# src: kubeadm-config.yaml.j2 +# dest: /root/kubeadm-config.yaml +# owner: root +# group: root +# mode: 600 +# when: +# - not groups['KubernetesMasters'][0] == ansible_hostname +# - server_enrolled.rc == 1 - name: Retreive certificats key on {{ lb_kubemaster }} - ansible.builtin.shell: set -o pipefail && kubeadm init phase upload-certs --upload-certs | grep -v upload-certs + shell: set -o pipefail && kubeadm init phase upload-certs --upload-certs | grep -v upload-certs register: kubernetes_certificateKey - check_mode: false delegate_to: "{{ lb_kubemaster }}" when: + - groups['KubernetesMasterConfigured'] is not defined - server_enrolled.rc == 1 - kubernetes_master|bool - name: Retreive token on "{{ lb_kubemaster }}" - ansible.builtin.command: kubeadm token create + command: kubeadm token create register: kubetoken delegate_to: "{{ lb_kubemaster }}" - check_mode: false when: + - groups['KubernetesMasterConfigured'] is not defined - server_enrolled.rc == 1 - name: Retreive hash certificat - ansible.builtin.shell: > + shell: > set -o pipefail && openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | @@ -333,21 +261,23 @@ sed 's/^.* //' register: cacerthash delegate_to: "{{ lb_kubemaster }}" - check_mode: false when: + - groups['KubernetesMasterConfigured'] is not defined - server_enrolled.rc == 1 - name: Deploy kubeadm config - ansible.builtin.template: + template: src: kubeadm-config.yaml.j2 dest: /root/kubeadm-config.yaml owner: root group: root mode: 0600 when: + - groups['KubernetesMasterConfigured'] is not defined - server_enrolled.rc == 1 - name: Join '{{ ansible_hostname }}' to Kubernetes cluster - ansible.builtin.command: kubeadm join --config=/root/kubeadm-config.yaml + command: kubeadm join --config=/root/kubeadm-config.yaml when: + - groups['KubernetesMasterConfigured'] is not defined - server_enrolled.rc == 1 diff --git a/tasks/install_server.yml b/tasks/install_server.yml index a9d0b9e..0dac5ad 100644 --- a/tasks/install_server.yml +++ b/tasks/install_server.yml @@ -1,53 +1,52 @@ --- -- name: Include vars for not taint Kubernetes masters - ansible.builtin.include_vars: masters.yml - when: - - kubernetes_master|bool - - not kubernetes_master_taint|bool - -- name: Add master to KubernetesMasters_ClusterName group - ansible.builtin.group_by: - key: KubernetesMasters_{{ kubernetes_cluster_name }} - check_mode: false - when: - - "'KubernetesMasters' in group_names" - -- name: Add node to KubernetesNodes_ClusterName group - ansible.builtin.group_by: - key: KubernetesNodes_{{ kubernetes_cluster_name }} - check_mode: false - when: - - "'KubernetesNodes' in group_names" - - - name: Disable SWAP since kubernetes can't work with swap enabled (1/2) - ansible.builtin.command: swapoff -a + command: swapoff -a changed_when: false - name: Remove swapfile from /etc/fstab (2/2) - ansible.posix.mount: + mount: name: swap fstype: swap state: absent - name: Create a thin pool for kubernetes - community.general.lvol: + lvol: vg: vg_sys thinpool: kubernetes - size: "{{ lv_kubernetes_size | default('20g') }}" + size: 20g -## Install API loadbalancer -# - ansible.builtin.include_tasks: "load_balancer.yml" -# when: -# - kubernetes_master|bool -# - groups['KubernetesMasters'] | length > 1 +# Install API loadbalancer +- include_tasks: "load_balancer.yml" + when: + - kubernetes_master|bool + - groups['KubernetesMasters'] | length > 1 + +- name: Audit policies directory + file: + path: "/etc/kubernetes/policies" + state: directory + owner: root + group: root + mode: 0700 + when: + - kubernetes_master|bool + +- name: Configure audit policy + copy: + src: "etc/kubernetes/policies/audit-policy.yaml" + dest: "/etc/kubernetes/policies/audit-policy.yaml" + group: root + owner: root + mode: 0644 + when: + - kubernetes_master|bool - name: Kubernetes cluster with kubeadm - ansible.builtin.include_tasks: "cluster_kubeadm.yml" + include_tasks: "cluster_kubeadm.yml" when: - kubernetes_cri != "k3s" - name: Kubernetes cluster with k3s - ansible.builtin.include_tasks: "cluster_k3s.yml" + include_tasks: "cluster_k3s.yml" when: - kubernetes_cri == "k3s" @@ -56,7 +55,7 @@ # - name: Make /root/.kube directory - ansible.builtin.file: + file: path: "/root/.kube" owner: root group: root @@ -66,10 +65,10 @@ - kubernetes_master|bool - name: Copy kubeconfig file from /etc/kubernetes/admin.conf - ansible.builtin.copy: + copy: src: "/etc/kubernetes/admin.conf" dest: /root/.kube/config - remote_src: true + remote_src: yes owner: root group: root mode: 0600 @@ -78,10 +77,10 @@ - kubernetes_cri != "k3s" - name: Copy kubeconfig file from /etc/rancher/k3s/k3s.yaml - ansible.builtin.copy: + copy: src: "/etc/rancher/k3s/k3s.yaml" dest: /root/.kube/config - remote_src: true + remote_src: yes owner: root group: root mode: 0600 @@ -89,40 +88,27 @@ - kubernetes_master|bool - kubernetes_cri == "k3s" -- name: Make link from /etc/rancher/k3s/k3s.yaml to /etc/kubernetes/admin.conf - file: - src: "/etc/rancher/k3s/k3s.yaml" - state: link - dest: "/etc/kubernetes/admin.conf" - force: yes - owner: root - group: root - when: - - kubernetes_master|bool - - kubernetes_cri == "k3s" - # # Manque autoconfig de .kube/config local # -# - name: Fetching CA certificat -# ansible.builtin.copy: -# src: /etc/kubernetes/pki/ca.crt -# dest: /root/.kube/{{ kubernetes_cluster_name }}/ca.crt -# when: -# - kubernetes_master|bigip_pool +#- name: Fetching CA certificat +# copy: +# src: /etc/kubernetes/pki/ca.crt +# dest: /root/.kube/{{ kubernetes_cluster_name }}/ca.crt +# when: +# - kubernetes_master|bigip_pool -# kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints --no-headers - name: Check if a node is still tainted - ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes '{{ ansible_hostname | lower }}' -o jsonpath='{.spec.taints}' + command: kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes '{{ ansible_hostname | lower }}' -o jsonpath='{.spec.taints}' register: current_taint - check_mode: false when: - kubernetes_master_taint|bool -- name: Taint the machine if needed -# ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master- - ansible.builtin.command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes '{{ ansible_hostname | lower }}' node-role.kubernetes.io/master- +- name: taint the machine if needed +# command: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master- + command: kubectl taint nodes '{{ ansible_hostname | lower }}' node-role.kubernetes.io/master- when: - kubernetes_master_taint|bool - current_taint.stdout + diff --git a/tasks/load_balancer.yml b/tasks/load_balancer.yml index d557058..aa9413d 100644 --- a/tasks/load_balancer.yml +++ b/tasks/load_balancer.yml @@ -1,21 +1,22 @@ --- - name: Install needed packages - ansible.builtin.package: - name: - - keepalived - - curl + package: + name: "{{ item }}" state: present - update_cache: true + update_cache: yes + with_items: + - keepalived + - curl notify: Restart keepalived - name: Install check_apiserver.sh script for keepalived - ansible.builtin.template: + template: src: etc/keepalived/check_apiserver.sh.j2 dest: /etc/keepalived/check_apiserver.sh owner: root group: root mode: 0755 - name: Install keepalived config file - ansible.builtin.template: + template: src: etc/keepalived/keepalived.conf.j2 dest: /etc/keepalived/keepalived.conf owner: root @@ -27,7 +28,7 @@ - groups['KubernetesMasters'][0] == ansible_hostname notify: Restart keepalived - name: Install keepalived config file - ansible.builtin.template: + template: src: etc/keepalived/keepalived.conf.j2 dest: /etc/keepalived/keepalived.conf owner: root @@ -38,6 +39,3 @@ when: - not groups['KubernetesMasters'][0] == ansible_hostname notify: Restart keepalived - - - name: Flush handlers - ansible.builtin.meta: flush_handlers diff --git a/tasks/main.yml b/tasks/main.yml index 9283b29..b57a12b 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,65 +1,57 @@ --- -- name: Kubernetes Install - tags: - - kubernetes - block: - - name: Include vars for {{ ansible_os_family }} - ansible.builtin.include_vars: "{{ ansible_os_family }}.yml" +- name: Include vars for {{ ansible_os_family }} + include_vars: "{{ ansible_os_family }}.yml" - - name: Define vars for master - ansible.builtin.set_fact: - kubernetes_server: true - kubernetes_master: true - kubernetes_master_taint: false - check_mode: false - when: - - "'KubernetesMasters' in group_names" - - "'KubernetesNodes' not in group_names" +- name: Define vars for master + set_fact: + kubernetes_server: true + kubernetes_master: true + kubernetes_master_taint: false + when: + - "'KubernetesMasters' in group_names" + - "'KubernetesNodes' not in group_names" - - name: Define vars for node - ansible.builtin.set_fact: - kubernetes_server: true - kubernetes_master: false - kubernetes_master_taint: false - check_mode: false - when: - - "'KubernetesNodes' in group_names" - - "'KubernetesMasters' not in group_names" +- name: Define vars for node + set_fact: + kubernetes_server: true + kubernetes_master: false + kubernetes_master_taint: false + when: + - "'KubernetesNodes' in group_names" + - "'KubernetesMasters' not in group_names" - - name: Define vars for taint master - ansible.builtin.set_fact: - kubernetes_server: true - kubernetes_master: true - kubernetes_master_taint: true - check_mode: false - when: - - "'KubernetesNodes' in group_names" - - "'KubernetesMasters' in group_names" +- name: Define vars for taint master + set_fact: + kubernetes_server: true + kubernetes_master: true + kubernetes_master_taint: true + when: + - "'KubernetesNodes' in group_names" + - "'KubernetesMasters' in group_names" - - name: Define vars for tooling - ansible.builtin.set_fact: - kubernetes_sever: false - check_mode: false - when: - - "'KubernetesMasters' not in group_names" - - "'KubernetesNodes' not in group_names" +- name: Define vars for tooling + set_fact: + kubernetes_sever: false + when: + - "'KubernetesMasters' not in group_names" + - "'KubernetesNodes' not in group_names" - - name: Install kubernetes rules for {{ ansible_os_family }} OS family - ansible.builtin.include_tasks: "{{ ansible_os_family }}.yml" +- name: Install kubernetes rules for {{ ansible_os_family }} OS family + include_tasks: "{{ ansible_os_family }}.yml" - # - name: Install kubernetes tools - # ansible.builtin.package: - # name: "{{ kubernetes_package_name }}" - # state: present - # update_cache: true - ## notify: Restart kubelet - # when: - # - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s") +- name: Install kubernetes tools + package: + name: "{{ kubernetes_package_name }}" + state: present + update_cache: yes +# notify: Restart kubelet + when: + - (not kubernetes_server|bool) or ( kubernetes_server|bool and kubernetes_cri != "k3s") - - name: Include kubernetes server rules - ansible.builtin.include_tasks: "install_server.yml" - when: - - kubernetes_server|bool +- name: Include kubernetes server rules + include_tasks: "install_server.yml" + when: + - kubernetes_server|bool - # - name: Install python library for docker - # package: name="{{ python_openshift_lib }}" state=latest update_cache=yes +#- name: Install python library for docker +# package: name="{{ python_openshift_lib }}" state=latest update_cache=yes diff --git a/templates/etc/firewalld/services/kubernetes.xml.j2 b/templates/etc/firewalld/services/kubernetes.xml.j2 index 95f0583..d4d0a53 100644 --- a/templates/etc/firewalld/services/kubernetes.xml.j2 +++ b/templates/etc/firewalld/services/kubernetes.xml.j2 @@ -9,12 +9,6 @@ {% if kubernetes_master == true %} # Kubernetes API server, used by all -{% endif %} -{% if kubernetes_cri == "k3s" %} -# K3S with flannel and wireguard - -{% else %} -{% if kubernetes_master == true %} # etcd server client API, used by kube-apiserver and etcd @@ -24,8 +18,8 @@ # kube-controler-manager, used by self -# Read-only Kubelet API (Deprecated) -# +# ??? + {% else %} {% endif %} @@ -41,5 +35,4 @@ {% endif %} -{% endif %} diff --git a/templates/etc/kubernetes/audit-webhook-kubeconfig.j2 b/templates/etc/kubernetes/audit-webhook-kubeconfig.j2 deleted file mode 100644 index 781d08c..0000000 --- a/templates/etc/kubernetes/audit-webhook-kubeconfig.j2 +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Config -clusters: -- name: falco - cluster: - server: http://$FALCO_SERVICE_CLUSTERIP:8765/k8s-audit -contexts: -- context: - cluster: falco - user: "" - name: default-context -current-context: default-context -preferences: {} -users: [] \ No newline at end of file diff --git a/templates/etc/rancher/k3s/config.yaml.j2 b/templates/etc/rancher/k3s/config.yaml.j2 deleted file mode 100644 index e55fb12..0000000 --- a/templates/etc/rancher/k3s/config.yaml.j2 +++ /dev/null @@ -1,55 +0,0 @@ -{% if 'Vpn' in group_names %} -flannel-backend: vxlan -{% else %} -flannel-backend: wireguard-native -{% endif %} -protect-kernel-defaults: true -{% if kubernetes_master|bool %} -secrets-encryption: true -kube-apiserver-arg: - - "enable-admission-plugins=NodeRestriction,AlwaysPullImages,EventRateLimit" - - 'admission-control-config-file=/etc/kubernetes/psa.yaml' - - 'audit-log-path=/var/log/apiserver/audit.log' - - 'audit-policy-file=/etc/kubernetes/policies/audit-policy.yaml' - - 'audit-log-maxage=30' - - 'audit-log-maxbackup=10' - - 'audit-log-maxsize=100' -# - "request-timeout=300s" -kube-controller-manager-arg: - - 'terminated-pod-gc-threshold=10' -cluster-cidr: - - {{ kubernetes_pods_network }} -service-cidr: - - {{ kubernetes_svc_network }} -{% if vars['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined %} -cluster-init: true -{% else %} -server: https://{{ kubernetes_master }}:6443 -token: ${NODE_TOKEN} -{% endif %} -{% else %} -server: https://{{ kubernetes_master }}:6443 -token: ${NODE_TOKEN} -{% endif %} -kubelet-arg: - - 'streaming-connection-idle-timeout=5m' - - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" -{% if false %} - - 'feature-gates=NodeSwap=true,CloudDualStackNodeIPs=true' -{% endif %} -{% if ansible_os_family == "RedHat" %} -selinux: true -{% endif %} -#embedded-registry: true -disable: - - traefik -{% if lookup('vars', 'ansible_' + kubernetes_interface ) != ansible_host %} -node-external-ip: {{ ansible_host }} -{% endif %} -node-ip: {{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }} -{% if false %} -# node-external-ip: 1.2.3.4 -#node-label: -# - "foo=bar" -# - "something=amazing" -{% endif %} diff --git a/templates/etc/sysconfig/kubelet.j2 b/templates/etc/sysconfig/kubelet.j2 index 92eaebd..b02129d 100644 --- a/templates/etc/sysconfig/kubelet.j2 +++ b/templates/etc/sysconfig/kubelet.j2 @@ -1,2 +1,2 @@ #https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates -KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}" +KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_default_ipv4.address }}" diff --git a/templates/etc/systemd/system/k3s.service.j2 b/templates/etc/systemd/system/k3s.service.j2 index 206c539..95ac5ae 100644 --- a/templates/etc/systemd/system/k3s.service.j2 +++ b/templates/etc/systemd/system/k3s.service.j2 @@ -7,9 +7,9 @@ After=network-online.target Type=notify EnvironmentFile=/etc/systemd/system/k3s.service.env {% if kubernetes_master|bool %} -ExecStart=/usr/local/bin/k3s server +ExecStart=/usr/local/bin/k3s server --flannel-backend=wireguard --disable traefik --secrets-encryption {% else %} -ExecStart=/usr/local/bin/k3s agent +ExecStart=/usr/local/bin/k3s agent --server https://{{ kubernetes_master }}:6443 --token ${NODE_TOKEN} {% endif %} KillMode=process Delegate=yes diff --git a/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 b/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 index fde66b0..e6f3596 100644 --- a/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 +++ b/templates/etc/systemd/system/kubelet.service.d/0-kubelet-extra-args.conf.j2 @@ -1,2 +1,2 @@ [Service] -Environment=KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}" +Environment=KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint={% if kubernetes_cri == "containerd" %}unix:///run/containerd/containerd.sock{% elif kubernetes_cri == "cri-o" %}unix:///var/run/crio/crio.sock{% endif %} --node-ip={{ ansible_default_ipv4.address }}" diff --git a/templates/etc/systemd/system/kubelet.service.d/11-cgroups.conf.j2 b/templates/etc/systemd/system/kubelet.service.d/11-cgroups.conf.j2 deleted file mode 100644 index 403ae0e..0000000 --- a/templates/etc/systemd/system/kubelet.service.d/11-cgroups.conf.j2 +++ /dev/null @@ -1,5 +0,0 @@ -# https://stackoverflow.com/a/57456786 -# https://stackoverflow.com/questions/57456667/failed-to-get-kubelets-cgroup -[Service] -CPUAccounting=true -MemoryAccounting=true diff --git a/templates/etc/systemd/system/kubelet.service.d/20-allow-swap.conf.j2 b/templates/etc/systemd/system/kubelet.service.d/20-allow-swap.conf.j2 deleted file mode 100644 index 3f54876..0000000 --- a/templates/etc/systemd/system/kubelet.service.d/20-allow-swap.conf.j2 +++ /dev/null @@ -1 +0,0 @@ -[Service] Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false" diff --git a/templates/etc/yum.repos.d/kubernetes.repo.j2 b/templates/etc/yum.repos.d/kubernetes.repo.j2 index b04037a..7ac0fdb 100644 --- a/templates/etc/yum.repos.d/kubernetes.repo.j2 +++ b/templates/etc/yum.repos.d/kubernetes.repo.j2 @@ -1,7 +1,7 @@ [kubernetes] name=Kubernetes baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-{{ ansible_machine }} -enabled=0 +enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg diff --git a/templates/kubeadm-config.yaml.j2 b/templates/kubeadm-config.yaml.j2 index 2adf101..9a95c08 100644 --- a/templates/kubeadm-config.yaml.j2 +++ b/templates/kubeadm-config.yaml.j2 @@ -1,4 +1,4 @@ -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration {% if kubetoken is defined %} bootstrapTokens: @@ -14,9 +14,8 @@ nodeRegistration: {% elif kubernetes_cri == "docker" %} criSocket: "/var/run/docker.sock" {% endif %} - name: {{ ansible_hostname }} {% if false %} - imagePullPolicy: IfNotPresent + name: "ec2-10-100-0-1" taints: - key: "kubeadmNode" value: "master" @@ -26,47 +25,70 @@ nodeRegistration: {% if ansible_service_mgr == "systemd" %} cgroup-driver: "systemd" {% endif %} + container-runtime: "remote" runtime-request-timeout: "5m" - node-ip: {{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }} +{% if kubernetes_cri == "containerd" %} + container-runtime-endpoint: "unix:///run/containerd/containerd.sock" +{% elif kubernetes_cri == "cri-o" %} + container-runtime-endpoint: "unix:///var/run/crio/crio.sock" +{% endif %} + node-ip: {{ ansible_default_ipv4.address }} read-only-port: "10255" ignorePreflightErrors: - SystemVerification -{% if (kubernetes_master|bool and not kubernetes_master_taint|bool) %} - - NumCPU -{% endif %} {% if true == false %} - IsPrivilegedUser {% endif %} localAPIEndpoint: - advertiseAddress: "{{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}" + advertiseAddress: "{{ ansible_default_ipv4.address }}" bindPort: 6443 -{% if kubernetes_master|bool and groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %} +{% if kubernetes_master|bool and groups['KubernetesMasterConfigured'] is defined %} certificateKey: "{{ kubernetes_certificateKey.stdout }}" {% endif %} -{% if kubernetes_master|bool and groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is not defined %} --- -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta2 +kind: JoinConfiguration +{% if kubernetes_master|bool %} +controlPlane: + localAPIEndpoint: + advertiseAddress: "{{ ansible_default_ipv4.address }}" + bindPort: 6443 +{% if groups['KubernetesMasterConfigured'] is defined %} + certificateKey: "{{ kubernetes_certificateKey.stdout }}" +{% endif %} +{% endif %} +discovery: + bootstrapToken: + apiServerEndpoint: "{{ lb_kubemaster }}:6443" +{% if groups['KubernetesMasterConfigured'] is defined %} + caCertHashes: + - sha256:{{ cacerthash.stdout }} + token: "{{ kubetoken.stdout }}" +{% endif %} +nodeRegistration: + kubeletExtraArgs: + node-ip: {{ ansible_default_ipv4.address }} + read-only-port: "10255" + ignorePreflightErrors: + - SystemVerification +--- +apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration kubernetesVersion: stable {% if lbip_kubeapiserver is defined %} controlPlaneEndpoint: "{{ lbip_kubeapiserver }}:6443" {% else %} -controlPlaneEndpoint: "{{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}:6443" +controlPlaneEndpoint: "{{ ansible_default_ipv4.address }}:6443" {% endif %} apiServer: extraArgs: - enable-admission-plugins: NodeRestriction,AlwaysPullImages,EventRateLimit + enable-admission-plugins: NodeRestriction,PodSecurityPolicy authorization-mode: "Node,RBAC" audit-policy-file: "/etc/kubernetes/policies/audit-policy.yaml" audit-log-path: "/var/log/apiserver/audit.log" audit-log-maxage: "30" audit-log-maxbackup: "10" audit-log-maxsize: "100" -{% if false %} -# Falco - audit-webhook-config-file: "/etc/kubernetes/policies/audit-webhook-kubeconfig" - audit-webhook-batch-max-wait: "5s" -{% endif %} extraVolumes: - name: "audit-log" hostPath: "/var/log/apiserver" @@ -74,163 +96,35 @@ apiServer: readOnly: false pathType: DirectoryOrCreate - name: "audit-policies" - hostPath: "/etc/kubernetes/policies/audit-policy.yaml" - mountPath: "/etc/kubernetes/policies/audit-policy.yaml" + hostPath: "/etc/kubernetes/policies" + mountPath: "/etc/kubernetes/policies" readOnly: false - pathType: File + pathType: DirectoryOrCreate {% if lb_kubemaster is defined %} certSANs: - "{{ lb_kubemaster }}" {% endif %} {% if kubernetes_network == "flannel" or kubernetes_network == "calico" %} networking: - podSubnet: "{{ kubernetes_pods_network }}" - serviceSubnet: "{{ kubernetes_svc_network }}" -{% endif %} -controllerManager: - extraArgs: - bind-address: 0.0.0.0 -scheduler: - extraArgs: - bind-address: 0.0.0.0 -etcd: - local: - dataDir: /var/lib/etcd - extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 -{% endif %} -{% if not kubernetes_master|bool or kubernetes_master|bool and groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %} ---- -apiVersion: kubeadm.k8s.io/v1beta3 -kind: JoinConfiguration -{% if kubernetes_master|bool %} -controlPlane: - localAPIEndpoint: - advertiseAddress: "{{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }}" - bindPort: 6443 -{% if groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %} - certificateKey: "{{ kubernetes_certificateKey.stdout }}" -{% endif %} -{% endif %} -discovery: - bootstrapToken: - apiServerEndpoint: "{{ lb_kubemaster }}:6443" -{% if groups['KubernetesMasterConfigured_' ~ kubernetes_cluster_name] is defined %} - caCertHashes: - - sha256:{{ cacerthash.stdout }} - token: "{{ kubetoken.stdout }}" -{% endif %} -nodeRegistration: - kubeletExtraArgs: - node-ip: {{ lookup('vars', 'ansible_' + kubernetes_interface ).ipv4.address }} -# read-only-port: "10255" - ignorePreflightErrors: - - SystemVerification -{% if kubernetes_swap_enabled is defined and kubernetes_swap_enabled|bool %} - - Swap -{% endif %} -{% if (kubernetes_master|bool and not kubernetes_master_taint|bool) %} - - NumCPU +{% if kubernetes_network == "flannel" %} + podSubnet: "10.244.0.0/16" +{% elif kubernetes_network == "calico" %} + podSubnet: "192.168.0.0/16" {% endif %} {% endif %} --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration -metricsBindAddress: "0.0.0.0:10249" {% if kubernetes_kubeproxy_mode is defined %} mode: {{ kubernetes_kubeproxy_mode }} -{% if kubernetes_kubeproxy_mode == "ipvs" %} -ipvs: - strictARP: true -{% endif %} {% endif %} --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration -#authentication: -# anonymous: -# enabled: false -# webhook: -# cacheTTL: 2m0s -# enabled: true -# x509: -# clientCAFile: /etc/kubernetes/pki/ca.crt -#authorization: -# mode: Webhook -# webhook: -# cacheAuthorizedTTL: 5m0s -# cacheUnauthorizedTTL: 30s +runtimeRequestTimeout: 5m {% if ansible_service_mgr == "systemd" %} cgroupDriver: systemd {% endif %} -#cgroupsPerQOS: true -#clusterDNS: -#- 10.96.0.10 -#clusterDomain: cluster.local -#configMapAndSecretChangeDetectionStrategy: Watch -#containerLogMaxFiles: 5 -#containerLogMaxSize: 10Mi -#contentType: application/vnd.kubernetes.protobuf -#cpuCFSQuota: true -#cpuCFSQuotaPeriod: 100ms -#cpuManagerPolicy: none -#cpuManagerReconcilePeriod: 10s -#enableControllerAttachDetach: true -#enableDebuggingHandlers: true -#enforceNodeAllocatable: -#- pods -#eventBurst: 10 -#eventRecordQPS: 5 -#evictionHard: -# imagefs.available: 15% -# memory.available: 500Mi -# nodefs.available: 10% -# nodefs.inodesFree: 5% -#evictionPressureTransitionPeriod: 5m0s -{% if kubernetes_swap_enabled is defined and kubernetes_swap_enabled|bool %} -# Activation du swap -failSwapOn: false -featureGates: - NodeSwap: true -memorySwap: - swapBehavior: UnlimitedSwap -{% endif %} -#fileCheckFrequency: 20s -#hairpinMode: promiscuous-bridge -#healthzBindAddress: 127.0.0.1 -#healthzPort: 10248 -#httpCheckFrequency: 20s -#imageGCHighThresholdPercent: 85 -#imageGCLowThresholdPercent: 80 -#imageMinimumGCAge: 2m0s -#iptablesDropBit: 15 -#iptablesMasqueradeBit: 14 -#kubeAPIBurst: 10 -#kubeAPIQPS: 5 -#logging: {} -#makeIPTablesUtilChains: true -#maxOpenFiles: 1000000 -#maxPods: 110 -#memorySwap: {} -#nodeLeaseDurationSeconds: 40 -#nodeStatusReportFrequency: 1m0s -#nodeStatusUpdateFrequency: 10s -#oomScoreAdj: -999 -#podPidsLimit: -1 -#port: 10250 -#registryBurst: 10 -#registryPullQPS: 5 -#resolvConf: /etc/resolv.conf -#rotateCertificates: true -runtimeRequestTimeout: 5m -#serializeImagePulls: true -#shutdownGracePeriod: 0s -#shutdownGracePeriodCriticalPods: 0s -#staticPodPath: /etc/kubernetes/manifests -#streamingConnectionIdleTimeout: 4h0m0s -#syncFrequency: 1m0s -#topologyManagerPolicy: none -#volumeStatsAggPeriod: 1m0s {% if false %} readOnlyPort: 1 diff --git a/templates/var/lib/rancher/k3s/server/manifests/np-00-intra-namespace.yaml.j2 b/templates/var/lib/rancher/k3s/server/manifests/np-00-intra-namespace.yaml.j2 deleted file mode 100644 index 8775180..0000000 --- a/templates/var/lib/rancher/k3s/server/manifests/np-00-intra-namespace.yaml.j2 +++ /dev/null @@ -1,12 +0,0 @@ -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: intra-namespace - namespace: kube-system -spec: - podSelector: {} - ingress: - - from: - - namespaceSelector: - matchLabels: - name: kube-system diff --git a/templates/var/lib/rancher/k3s/server/manifests/np-01-default-network-dns-policy.yaml.j2 b/templates/var/lib/rancher/k3s/server/manifests/np-01-default-network-dns-policy.yaml.j2 deleted file mode 100644 index 9357b4f..0000000 --- a/templates/var/lib/rancher/k3s/server/manifests/np-01-default-network-dns-policy.yaml.j2 +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-network-dns-policy - namespace: kube-system -spec: - ingress: - - ports: - - port: 53 - protocol: TCP - - port: 53 - protocol: UDP - podSelector: - matchLabels: - k8s-app: kube-dns - policyTypes: - - Ingress diff --git a/templates/var/lib/rancher/k3s/server/manifests/np-03-metrics-server-traefik.yaml.j2 b/templates/var/lib/rancher/k3s/server/manifests/np-03-metrics-server-traefik.yaml.j2 deleted file mode 100644 index e7b8621..0000000 --- a/templates/var/lib/rancher/k3s/server/manifests/np-03-metrics-server-traefik.yaml.j2 +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-metrics-server - namespace: kube-system -spec: - podSelector: - matchLabels: - k8s-app: metrics-server - ingress: - - {} - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-svclbtraefik-ingress - namespace: kube-system -spec: - podSelector: - matchLabels: - svccontroller.k3s.cattle.io/svcname: traefik - ingress: - - {} - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-traefik-v121-ingress - namespace: kube-system -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: traefik - ingress: - - {} - policyTypes: - - Ingress ---- diff --git a/vars/RedHat.yml b/vars/RedHat.yml index 4eecd17..a5905f3 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -1,9 +1,8 @@ --- kubernetes_package_name: - - kubectl-{{ kubernetes_version }} - - kubelet-{{ kubernetes_version }} - - kubeadm-{{ kubernetes_version }} + - kubectl + - kubelet + - kubeadm - iproute-tc - - ipvsadm #kubernetes_remove_packages_name: # - kubernetes.io diff --git a/vars/masters.yml b/vars/masters.yml deleted file mode 100644 index 4a03a33..0000000 --- a/vars/masters.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -lv_containers_size: 2g -lv_kubernetes_size: 8g \ No newline at end of file