configure for ipv6 dual stack

ansible/inventory/inventory.yaml

@@ -37,6 +37,11 @@ all:
             kube01:
             # kube02:
             # kube03:  
+        k8s_storage:
+          hosts:
+            kube01:
+            # kube02:
+            # kube03:      
     firewalls:
       children:
         fortigate:

ansible/playbooks/05_k8s_deploy.yaml

@@ -25,4 +25,4 @@
   gather_facts: false
   become: false
   roles:
-    - k8s_storage_deploy
+    - k8s_storage_ebs_deploy

ansible/roles/k8s_control/defaults/main.yaml

@@ -1,10 +1,13 @@
 ---
 k8s_control_master_node: kube01.balsillie.net
 k8s_endpoint: k8s.balsillie.net
-k8s_address: "192.168.199.240"
+k8s_v4_address: "192.168.199.240"
+k8s_v6_address: "2a01:4f8:13b:f201::0240"
 k8s_api_port: "6443"
 k8s_cri_socket: /run/containerd/containerd.sock
 k8s_service_domain: cluster.internal
-k8s_pod_cidr: 10.128.0.0/16
+k8s_v4_pod_cidr: 10.128.0.0/16
-k8s_service_cidr: 10.96.0.0/16
+k8s_v6_pod_cidr: 2a01:4f8:13b:f202::00/64
+k8s_v4_service_cidr: 10.96.0.0/16
+k8s_v6_service_cidr: 2a01:4f8:13b:f203::00/64
 local_user: michael

ansible/roles/k8s_control/tasks/main.yaml

@@ -1,9 +1,12 @@
 ---
 - name: write cluster api address to hosts file
   ansible.builtin.lineinfile:
-    line: "{{ k8s_address }} {{ k8s_endpoint }}"
+    line: "{{ item }}"
     insertafter: EOF
     path: /etc/hosts
+  with_items:
+    - "{{ k8s_v4_address }} {{ k8s_endpoint }}"
+    - "{{ k8s_v6_address }} {{ k8s_endpoint }}"
 
 - name: create containerd config dir
   ansible.builtin.file:
@@ -42,6 +45,19 @@
     state: started
     enabled: true
 
+- name: ensure ip forwarding is active
+  ansible.posix.sysctl:
+    name: "{{ item.key }}"
+    value: "{{ item.value }}"
+    state: present
+    sysctl_set: true
+    reload: true
+  with_items:
+    - key: net.ipv6.conf.all.forwarding
+      value: '1'
+    - key: net.ipv4.conf.all.forwarding
+      value: '1'
+
 - name: kubeadm init master node
   when: ansible_host == hostvars[groups['k8s_control'][0]]['ansible_host']
   block:
@@ -50,15 +66,16 @@
     ansible.builtin.shell:
       cmd: |
         kubeadm init \
-        --control-plane-endpoint {{ k8s_address }} \
+        --control-plane-endpoint {{ k8s_endpoint }} \
         --cri-socket /run/containerd/containerd.sock \
-        --pod-network-cidr {{ k8s_pod_cidr }} \
+        --pod-network-cidr {{ k8s_v4_pod_cidr }},{{ k8s_v6_pod_cidr }} \
-        --service-cidr {{ k8s_service_cidr }} \
+        --service-cidr {{ k8s_v4_service_cidr }},{{ k8s_v6_service_cidr}} \
         --apiserver-bind-port {{ k8s_api_port }} \
         --apiserver-cert-extra-sans {{ k8s_endpoint }} \
-        --apiserver-cert-extra-sans {{ k8s_address }} \
+        --apiserver-cert-extra-sans {{ k8s_v4_ddress }} \
+        --apiserver-cert-extra-sans {{ k8s_v6_ddress }} \
         --node-name {{ ansible_hostname }} \
-        --feature-gates IPv6DualStack=false \
+        --feature-gates IPv6DualStack=true \
         --feature-gates PublicKeysECDSA=true \
         --service-dns-domain {{ k8s_service_domain }} \
       creates: /etc/kubernetes/admin.conf

ansible/roles/k8s_network/defaults/main.yaml

@@ -6,8 +6,12 @@ k8s_network_packages:
   - python-kubernetes
   - python-yaml
   - python-jsonpatch
-k8s_pod_cidr: 10.128.0.0/16
+k8s_v4_pod_cidr: 10.128.0.0/16
-k8s_network_blocksize: 20
+k8s_v6_pod_cidr: 2a01:4f8:13b:f202::00/64
+k8s_v4_service_cidr: 10.96.0.0/16
+k8s_v6_service_cidr: 2a01:4f8:13b:f203::00/64
+k8s_v4_network_blocksize: 20
+k8s_v6_network_blocksize: 120
 k8s_network_encapsulation: None
 k8s_network_nat: Disabled
 k8s_network_bgp: Enabled

ansible/roles/k8s_network/tasks/main.yaml

@@ -41,13 +41,25 @@
   ansible.utils.update_fact:
     updates:
       - path: calico_default_installation.spec.calicoNetwork.ipPools[0].blockSize
-        value: "{{ k8s_network_blocksize }}"
+        value: "{{ k8s_v4_network_blocksize }}"
       - path: calico_default_installation.spec.calicoNetwork.ipPools[0].cidr
-        value: "{{ k8s_pod_cidr }}"
+        value: "{{ k8s_v4_pod_cidr }}"
       - path: calico_default_installation.spec.calicoNetwork.ipPools[0].encapsulation
         value: "{{ k8s_network_encapsulation }}"
       - path: calico_default_installation.spec.calicoNetwork.ipPools[0].natOutgoing
         value: "{{ k8s_network_nat }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[0].nodeSelector
+        value: "all()"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[1].blockSize
+        value: "{{ k8s_v6_network_blocksize }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[1].cidr
+        value: "{{ k8s_v6_pod_cidr }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[1].encapsulation
+        value: "{{ k8s_network_encapsulation }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[1].natOutgoing
+        value: "{{ k8s_network_nat }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[1].nodeSelector
+        value: "all()"
       - path: calico_default_installation.spec.calicoNetwork.bgp
         value: "{{ k8s_network_bgp }}"
       - path: calico_default_installation.spec.calicoNetwork.linuxDataplane

ansible/roles/k8s_storage_ebs_deploy/tasks/disks.yaml

@@ -0,0 +1,53 @@
+---
+- name: setup disks on storage nodes
+  delegate_to: "{{ node }}"
+  delegate_facts: true
+  block:
+
+    - name: ensure parted is installed
+      community.general.pacman:
+        name: parted
+        state: latest
+        update_cache: false
+      when:
+        - ansible_os_family == 'Archlinux'
+
+    - name: create containerd image partition
+      community.general.parted:
+        device: "{{ item.disk }}"
+        align: optimal
+        name: "{{ item.part }}"
+        label: gpt
+        number: 1
+        part_start: 0%
+        part_end: 100%
+        state: present
+        fs_type: ext4
+      with_items:
+        - disk: /dev/vdc
+          part: ebs-ssd
+        - disk: /dev/vdd
+          part: ebs-hdd
+
+    - name: create containerd partition filesystem
+      community.general.filesystem:
+        dev: "{{ item }}"
+        fstype: ext4
+        resizefs: true
+        state: present
+      with_items:
+        - /dev/disk/by-partlabel/ebs-ssd
+        - /dev/disk/by-partlabel/ebs-hdd
+
+    - name: mount ebs disks
+      ansible.posix.mount:
+        state: mounted
+        src: "{{ item.src }}"
+        path: "{{ item.path }}"
+        fstype: ext4
+        boot: true
+      with_items:
+        - src: /dev/disk/by-partlabel/ebs-ssd
+          path: /ebs/ssd
+        - src: /dev/disk/by-partlabel/ebs-hdd
+          path: /ebs/hdd

ansible/roles/k8s_storage_ebs_deploy/tasks/main.yaml

@@ -1,4 +1,12 @@
 ---
+- name: setup disks for ebs
+  ansible.builtin.include_tasks:
+    file: disks.yaml
+  with_items:
+    - "{{ groups['k8s_storage'] }}"
+  loop_control:
+    loop_var: node
+
 - name: create target directory for open-ebs files
   become: false
   ansible.builtin.file:

ansible/roles/k8s_storage_ebs_deploy/tasks/nodes.yaml

@@ -1,60 +0,0 @@
----
-# - name: create mount directories for ebs drives
-#   ansible.builtin.file:
-#     path: "{{ item }}"
-#     state: directory
-#     owner: root
-#     group: root
-#     mode: 0775
-#   with_items:
-#     - /ebs
-#     - /ebs/ssd
-#     - /ebs/hdd
-
-- name: ensure parted is installed
-  community.general.pacman:
-    name: parted
-    state: latest
-    update_cache: true
-  when:
-    - ansible_os_family == 'Archlinux'
-
-- name: create containerd image partition
-  community.general.parted:
-    device: "{{ item.disk }}"
-    align: optimal
-    name: "{{ item.part }}"
-    label: gpt
-    number: 1
-    part_start: 0%
-    part_end: 100%
-    state: present
-    fs_type: ext4
-  with_items:
-    - disk: /dev/vdc
-      part: ebs-ssd
-    - disk: /dev/vdd
-      part: ebs-hdd
-
-- name: create containerd partition filesystem
-  community.general.filesystem:
-    dev: "{{ item }}"
-    fstype: ext4
-    resizefs: true
-    state: present
-  with_items:
-    - /dev/disk/by-partlabel/ebs-ssd
-    - /dev/disk/by-partlabel/ebs-hdd
-
-- name: mount ebs disks
-  ansible.posix.mount:
-    state: mounted
-    src: "{{ item.src }}"
-    path: "{{ item.path }}"
-    fstype: ext4
-    boot: true
-  with_items:
-    - src: /dev/disk/by-partlabel/ebs-ssd
-      path: /ebs/ssd
-    - src: /dev/disk/by-partlabel/ebs-hdd
-      path: /ebs/hdd