tuning

ansible/inventory/inventory.yaml

@@ -25,18 +25,18 @@ all:
         k8s_control:
           hosts:
             kube01:
-            kube02:
+            # kube02:
-            kube03:
+            # kube03:
         k8s_taint:
           hosts:
             kube01:
-            kube02:
+            # kube02:
-            kube03:
+            # kube03:
         k8s_worker:
           hosts:
             kube01:
-            kube02:
+            # kube02:
-            kube03:       
+            # kube03:       
     firewalls:
       children:
         fortigate:

ansible/playbooks/05_k8s_deploy.yaml

@@ -25,4 +25,4 @@
   gather_facts: false
   become: false
   roles:
-    - k8s_storage
+    - k8s_storage_deploy

ansible/roles/k8s_control/tasks/main.yaml

@@ -50,20 +50,24 @@
     ansible.builtin.shell:
       cmd: |
         kubeadm init \
-        --control-plane-endpoint {{ k8s_endpoint }} \
+        --control-plane-endpoint {{ k8s_address }} \
         --cri-socket /run/containerd/containerd.sock \
         --pod-network-cidr {{ k8s_pod_cidr }} \
         --service-cidr {{ k8s_service_cidr }} \
         --apiserver-bind-port {{ k8s_api_port }} \
         --apiserver-cert-extra-sans {{ k8s_endpoint }} \
+        --apiserver-cert-extra-sans {{ k8s_address }} \
         --node-name {{ ansible_hostname }} \
         --feature-gates IPv6DualStack=false \
         --feature-gates PublicKeysECDSA=true \
-        --skip-phases=addon/kube-proxy \
         --service-dns-domain {{ k8s_service_domain }} \
       creates: /etc/kubernetes/admin.conf
     register: k8s_init
 
+            # --skip-phases=addon/kube-proxy \
+
+# TODO a check here to wait until node has finished init
+
   - name: register the control plane certificate key
     ansible.builtin.shell:
       cmd: |
@@ -109,12 +113,13 @@
       --control-plane \
       --certificate-key {{ hostvars[groups['k8s_control'][0]]['join_key']['stdout_lines'][2] }} \
       --cri-socket /run/containerd/containerd.sock \
-      --skip-phases=addon/kube-proxy \
       --node-name {{ ansible_hostname }}
     creates: /etc/kubernetes/admin.conf
 
+      # --skip-phases=addon/kube-proxy \    
+
 - name: set kubelet service to enabled at startup
   ansible.builtin.service:
     name: kubelet.service
-    state: running
+    state: started
     enabled: true

ansible/roles/k8s_network/defaults/main.yaml

@@ -11,10 +11,11 @@ k8s_network_blocksize: 20
 k8s_network_encapsulation: None
 k8s_network_nat: Disabled
 k8s_network_bgp: Enabled
-k8s_network_dataplane: BPF
+k8s_network_dataplane: Iptables # BPF or Iptables
-k8s_network_hostports: Disabled
+k8s_network_hostports: Enabled # Disabled when dataplne is BPF
 k8s_network_bgp_peer_name: opnsense
 k8s_network_bgp_peer_address: 192.168.199.254
 k8s_network_bgp_peer_as: 64612
 k8s_endpoint: k8s.balsillie.net
 k8s_api_port: "6443"
+k8s_address: "192.168.199.240"

ansible/roles/k8s_network/tasks/main.yaml

@@ -56,17 +56,27 @@
         value: "{{ k8s_network_hostports }}"        
   register: calico_updated_installation      
 
-- name: add config map for ebpf mode # https://projectcalico.docs.tigera.io/maintenance/ebpf/install
+- name: create tigera operator namespace
   ansible.builtin.set_fact:
-    calico_configmap_ebpf:
+    calico_namespace:
-      kind: ConfigMap
       apiVersion: v1
+      kind: Namespace
       metadata:
-        name: kubernetes-services-endpoint
+        name: tigera-operator
-        namespace: tigera-operator
+        labels:
-      data:
+          name: tigera-operator
-        KUBERNETES_SERVICE_HOST: "{{ k8s_endpoint }}"
+
-        KUBERNETES_SERVICE_PORT: "{{ k8s_api_port }}"
+# - name: add config map for ebpf mode # https://projectcalico.docs.tigera.io/maintenance/ebpf/install
+#   ansible.builtin.set_fact:
+#     calico_configmap_ebpf:
+#       kind: ConfigMap
+#       apiVersion: v1
+#       metadata:
+#         name: kubernetes-services-endpoint
+#         namespace: tigera-operator
+#       data:
+#         KUBERNETES_SERVICE_HOST: "{{ k8s_address }}"
+#         KUBERNETES_SERVICE_PORT: "{{ k8s_api_port }}"
 
 - name: add bgp peer for gateway/router
   ansible.builtin.set_fact:
@@ -79,10 +89,15 @@
         peerIP: "{{ k8s_network_bgp_peer_address }}"
         asNumber: "{{ k8s_network_bgp_peer_as }}"
 
-- name: write out calico configmap for ebpf mode
+- name: write out calico namespace crd
   ansible.builtin.copy:
-    content: "{{ calico_configmap_ebpf | to_nice_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element
+    content: "{{ calico_namespace | to_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element
-    dest: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml"
+    dest: "{{ ansible_search_path[0] }}/files/calico/calico_namespace.yaml"
+
+# - name: write out calico configmap for ebpf mode
+#   ansible.builtin.copy:
+#     content: "{{ calico_configmap_ebpf | to_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element
+#     dest: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml"
 
 - name: write out calico installation definition
   ansible.builtin.copy:
@@ -97,26 +112,20 @@
 # TODO two api versions exist for BGP, one only becomes available after calico is online. Do they both work??
 # crd.projectcalico.org/v1
 # projectcalico.org/v3
+
 - name: write out calico bgp peer definition
   ansible.builtin.copy:
     content: "{{ calico_bgp_peer | to_nice_yaml }}"
     dest: "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml"
 
-- name: install configmap for ebpf mode to cluster
+- name: install calico definitions to cluster # The order here matters. namespace > config map > operator > installation
-  kubernetes.core.k8s:
-    src: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml"
-    state: present
-
-- name: install calico operator to cluster
-  kubernetes.core.k8s:
-    src: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml"
-    state: present
-
-- name: install calico definitions to cluster
   kubernetes.core.k8s:
     state: present
     src: "{{ item }}"
   with_items:
+      # - "{{ ansible_search_path[0] }}/files/calico/calico_namespace.yaml"
+      # - "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml"
+      - "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml"
       - "{{ ansible_search_path[0] }}/files/calico/calico_installation.yaml"
       - "{{ ansible_search_path[0] }}/files/calico/calico_apiserver.yaml"
       - "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml"

ansible/roles/k8s_storage_deploy/tasks/main.yaml

@@ -1,58 +1,58 @@
 ---
-# - name: create target directory for rook files
+- name: create target directory for rook files
-#   ansible.builtin.file:
+  ansible.builtin.file:
-#     path: "{{ ansible_search_path[0] }}/files/rook"
+    path: "{{ ansible_search_path[0] }}/files/rook"
-#     state: directory
+    state: directory
-#     mode: 0775
+    mode: 0775
 
-# - name: load rbd kernel module
+- name: load rbd kernel module
-#   become: true
+  become: true
-#   delegate_to: "{{ item }}"
+  delegate_to: "{{ item }}"
-#   with_items: "{{ groups['k8s_worker'] }}"
+  with_items: "{{ groups['k8s_worker'] }}"
-#   community.general.modprobe:
+  community.general.modprobe:
-#     name: rbd
+    name: rbd
-#     state: present
+    state: present
   
-# - name: set rbd kernel module to load at boot
+- name: set rbd kernel module to load at boot
-#   become: true
+  become: true
-#   delegate_to: "{{ item }}"
+  delegate_to: "{{ item }}"
-#   with_items: "{{ groups['k8s_worker'] }}"
+  with_items: "{{ groups['k8s_worker'] }}"
-#   ansible.builtin.copy:
+  ansible.builtin.copy:
-#     dest: /etc/modules-load.d/rbd.conf
+    dest: /etc/modules-load.d/rbd.conf
-#     content: rbd
+    content: rbd
-#     owner: root
+    owner: root
-#     group: root
+    group: root
-#     mode: 0660
+    mode: 0660
 
-# - name: install lvm2 package
+- name: install lvm2 package
-#   become: true
+  become: true
-#   delegate_to: "{{ item }}"
+  delegate_to: "{{ item }}"
-#   with_items: "{{ groups['k8s_worker'] }}"
+  with_items: "{{ groups['k8s_worker'] }}"
-#   community.general.pacman:
+  community.general.pacman:
-#     name: lvm2
+    name: lvm2
-#     state: latest
+    state: latest
-#     update_cache: true
+    update_cache: true
 
-# - name: download the rook manifests
+- name: download the rook manifests
-#   ansible.builtin.uri:
+  ansible.builtin.uri:
-#     url: https://raw.githubusercontent.com/rook/rook/{{ rook_version }}/deploy/examples/{{ item }}.yaml
+    url: https://raw.githubusercontent.com/rook/rook/{{ rook_version }}/deploy/examples/{{ item }}.yaml
-#     dest: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml"
+    dest: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml"
-#     creates: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml"
+    creates: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml"
-#     mode: 0664
+    mode: 0664
-#   with_items:
+  with_items:
-#     - crds
+    - crds
-#     - common
+    - common
-#     - operator
+    - operator
-#     - cluster
+    - cluster
 
-# - name: deploy the rook manifest # The order of the items is important, crds > common > operator , see https://github.com/rook/rook/blob/v1.10.4/deploy/examples/common.yaml
+- name: deploy the rook manifest # The order of the items is important, crds > common > operator , see https://github.com/rook/rook/blob/v1.10.4/deploy/examples/common.yaml
-#   kubernetes.core.k8s:
+  kubernetes.core.k8s:
-#     src: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml"
+    src: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml"
-#     state: present
+    state: present
-#   with_items:
+  with_items:
-#     - crds
+    - crds
-#     - common
+    - common
-#     - operator
+    # - operator
 
 # # TODO somehow turn this command:
 # # kubectl -n rook-ceph get pod -o json | jq '.items[].status.containerStatuses[].ready'
@@ -92,10 +92,10 @@
 
 # TODO create a check and wait until cluster is created and running
 
-- name: create the storage providers
+# - name: create the storage providers
-  kubernetes.core.k8s:
+#   kubernetes.core.k8s:
-    src: "{{ ansible_search_path[0] }}/files/config/{{ item }}"
+#     src: "{{ ansible_search_path[0] }}/files/config/{{ item }}"
-    state: present
+#     state: present
-  with_items:
+#   with_items:
-    - blockpool_ssd_replica.yaml
+#     - blockpool_ssd_replica.yaml
-    - filesystem_multi.yaml
+#     - filesystem_multi.yaml