cluster networking operational

.gitignore

@@ -1,2 +1,3 @@
 ansible/vault_password
 ansible/inventory/host_vars/*/vault.yml
+ansible/roles/k8s_network/files/calico

ansible/playbooks/04_k8s.yaml

@@ -1,7 +1,14 @@
 ---
-- name: configure control plane
+# - name: configure control plane
-  hosts: k8s_control
+#   hosts: k8s_control
+#   gather_facts: true
+#   become: true
+#   roles:
+#     - k8s_control
+
+- name: configure calico networking
+  hosts: localhost
   gather_facts: true
-  become: true
+  become: false
   roles:
-    - k8s_control
+    - k8s_network

ansible/playbooks/88_scratch.yaml

@@ -1,7 +1,7 @@
 ---
 - name: scratch testing
-  hosts: localhost
+  hosts: k8s_control
-  gather_facts: false
+  gather_facts: true
-  become: false
+  become: true
   roles:
     - scratch

ansible/roles/k8s_control/defaults/main.yaml

@@ -1,6 +1,7 @@
 ---
 k8s_control_master_node: kube01.balsillie.net
 k8s_endpoint: k8s.balsillie.net
+k8s_address: "192.168.199.240"
 k8s_api_port: "6443"
 k8s_cri_socket: /run/containerd/containerd.sock
 k8s_service_domain: cluster.internal

ansible/roles/k8s_control/tasks/main.yaml

@@ -1,4 +1,10 @@
 ---
+- name: write cluster api address to hosts file
+  ansible.builtin.lineinfile:
+    line: "{{ k8s_address }} {{ k8s_endpoint }}"
+    insertafter: EOF
+    path: /etc/hosts
+
 - name: create containerd config dir
   ansible.builtin.file:
     path: /etc/containerd
@@ -53,6 +59,7 @@
         --node-name {{ ansible_hostname }} \
         --feature-gates IPv6DualStack=false \
         --feature-gates PublicKeysECDSA=true \
+        --skip-phases=addon/kube-proxy \
         --service-dns-domain {{ k8s_service_domain }} \
       creates: /etc/kubernetes/admin.conf
     register: k8s_init
@@ -102,5 +109,6 @@
       --control-plane \
       --certificate-key {{ hostvars[groups['k8s_control'][0]]['join_key']['stdout_lines'][2] }} \
       --cri-socket /run/containerd/containerd.sock \
+      --skip-phases=addon/kube-proxy \
       --node-name {{ ansible_hostname }}
     creates: /etc/kubernetes/admin.conf

ansible/roles/k8s_network/defaults/main.yaml

@@ -0,0 +1,20 @@
+---
+calico_version: v3.24.3
+k8s_network_packages:
+  - kubernetes-tools
+  - kubectl-plugins
+  - python-kubernetes
+  - python-yaml
+  - python-jsonpatch
+k8s_pod_cidr: 10.128.0.0/16
+k8s_network_blocksize: 20
+k8s_network_encapsulation: None
+k8s_network_nat: Disabled
+k8s_network_bgp: Enabled
+k8s_network_dataplane: BPF
+k8s_network_hostports: Disabled
+k8s_network_bgp_peer_name: opnsense
+k8s_network_bgp_peer_address: 192.168.199.254
+k8s_network_bgp_peer_as: 64612
+k8s_endpoint: k8s.balsillie.net
+k8s_api_port: "6443"

ansible/roles/k8s_network/tasks/main.yaml

@@ -0,0 +1,119 @@
+---
+- name: ensure required python bindings are present
+  when: ansible_os_family == 'Archlinux'
+  become: true
+  community.general.pacman:
+    name: "{{ k8s_network_packages }}"
+    state: latest
+    update_cache: true
+
+- name: create target directory for calico files
+  ansible.builtin.file:
+    path: "{{ ansible_search_path[0] }}/files/calico"
+    state: directory
+    mode: 0775
+
+- name: download the calico operator manifest
+  ansible.builtin.uri:
+    url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/tigera-operator.yaml
+    dest: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml"
+    creates: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml"
+    mode: 0664
+
+- name: download calico configuration
+  ansible.builtin.uri:
+    url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/custom-resources.yaml
+    dest: "{{ ansible_search_path[0] }}/files/calico/calico_resources_{{ calico_version }}.yaml"
+    creates: "{{ ansible_search_path[0] }}/files/calico/calico_resources_{{ calico_version }}.yaml"
+    mode: 0664
+
+- name: read the default config into memory
+  ansible.builtin.slurp:
+    src: "{{ ansible_search_path[0] }}/files/calico/calico_resources_{{ calico_version }}.yaml"
+  register: calico_file_raw
+
+- name: split and parse calico settings from the file data # to_yaml will reject the --- in the original manifest, hence data must be split.
+  ansible.builtin.set_fact:
+    calico_default_installation: "{{ (calico_file_raw['content'] | b64decode).split(\"---\")[0] | from_yaml }}"
+    calico_default_apiserver: "{{ (calico_file_raw['content'] | b64decode).split(\"---\")[1] | from_yaml }}"
+
+- name: update calico installation settings to desired values
+  ansible.utils.update_fact:
+    updates:
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[0].blockSize
+        value: "{{ k8s_network_blocksize }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[0].cidr
+        value: "{{ k8s_pod_cidr }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[0].encapsulation
+        value: "{{ k8s_network_encapsulation }}"
+      - path: calico_default_installation.spec.calicoNetwork.ipPools[0].natOutgoing
+        value: "{{ k8s_network_nat }}"
+      - path: calico_default_installation.spec.calicoNetwork.bgp
+        value: "{{ k8s_network_bgp }}"
+      - path: calico_default_installation.spec.calicoNetwork.linuxDataplane
+        value: "{{ k8s_network_dataplane }}"
+      - path: calico_default_installation.spec.calicoNetwork.hostPorts
+        value: "{{ k8s_network_hostports }}"        
+  register: calico_updated_installation      
+
+- name: add config map for ebpf mode # https://projectcalico.docs.tigera.io/maintenance/ebpf/install
+  ansible.builtin.set_fact:
+    calico_configmap_ebpf:
+      kind: ConfigMap
+      apiVersion: v1
+      metadata:
+        name: kubernetes-services-endpoint
+        namespace: tigera-operator
+      data:
+        KUBERNETES_SERVICE_HOST: "{{ k8s_endpoint }}"
+        KUBERNETES_SERVICE_PORT: "{{ k8s_api_port }}"
+
+- name: add bgp peer for gateway/router
+  ansible.builtin.set_fact:
+    calico_bgp_peer:
+      apiVersion: crd.projectcalico.org/v1
+      kind: BGPPeer
+      metadata:
+        name: "{{ k8s_network_bgp_peer_name }}"
+      spec:
+        peerIP: "{{ k8s_network_bgp_peer_address }}"
+        asNumber: "{{ k8s_network_bgp_peer_as }}"
+
+- name: write out calico configmap for ebpf mode
+  ansible.builtin.copy:
+    content: "{{ calico_configmap_ebpf | to_nice_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element
+    dest: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml"
+
+- name: write out calico installation definition
+  ansible.builtin.copy:
+    content: "{{ calico_updated_installation.calico_default_installation | to_nice_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element
+    dest: "{{ ansible_search_path[0] }}/files/calico/calico_installation.yaml"
+
+- name: write out calico apiserver definition
+  ansible.builtin.copy:
+    content: "{{ calico_default_apiserver | to_nice_yaml }}"
+    dest: "{{ ansible_search_path[0] }}/files/calico/calico_apiserver.yaml"
+
+- name: write out calico bgp peer definition
+  ansible.builtin.copy:
+    content: "{{ calico_bgp_peer | to_nice_yaml }}"
+    dest: "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml"
+
+- name: install configmap for ebpf mode to cluster
+  kubernetes.core.k8s:
+    src: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml"
+    state: present
+
+- name: install calico operator to cluster
+  kubernetes.core.k8s:
+    src: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml"
+    state: present
+
+- name: install calico definitions to cluster
+  kubernetes.core.k8s:
+    state: present
+    src: "{{ item }}"
+  with_items:
+      - "{{ ansible_search_path[0] }}/files/calico/calico_installation.yaml"
+      - "{{ ansible_search_path[0] }}/files/calico/calico_apiserver.yaml"
+      - "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml"

ansible/roles/scratch/tasks/main.yaml

@@ -1,8 +1,6 @@
 ---
 - name: debug hostname
-  ansible.builtin.debug:
+  ansible.builtin.lineinfile:
-    msg: 
+    line: 192.168.199.240 k8s.balsillie.net
-      - "{{ hostvars[groups['k8s_control'][0]]['ansible_host'] }}"
+    insertafter: EOF
-      - "{{ ansible_host }}"
+    path: /etc/hosts
-
-    # ['hosts'][0]