From 6020b9771ca7b735c697a48d3eef13999fc1880c Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 1 Nov 2022 22:13:56 +1300 Subject: [PATCH] cluster networking operational --- .gitignore | 3 +- ansible/playbooks/04_k8s.yaml | 15 ++- ansible/playbooks/88_scratch.yaml | 6 +- ansible/roles/k8s_control/defaults/main.yaml | 1 + ansible/roles/k8s_control/tasks/main.yaml | 8 ++ ansible/roles/k8s_network/defaults/main.yaml | 20 ++++ ansible/roles/k8s_network/tasks/main.yaml | 119 +++++++++++++++++++ ansible/roles/scratch/tasks/main.yaml | 10 +- 8 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 ansible/roles/k8s_network/defaults/main.yaml create mode 100644 ansible/roles/k8s_network/tasks/main.yaml diff --git a/.gitignore b/.gitignore index 83f7a8a..7369562 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ ansible/vault_password -ansible/inventory/host_vars/*/vault.yml \ No newline at end of file +ansible/inventory/host_vars/*/vault.yml +ansible/roles/k8s_network/files/calico \ No newline at end of file diff --git a/ansible/playbooks/04_k8s.yaml b/ansible/playbooks/04_k8s.yaml index 982ed9a..ccfac72 100644 --- a/ansible/playbooks/04_k8s.yaml +++ b/ansible/playbooks/04_k8s.yaml @@ -1,7 +1,14 @@ --- -- name: configure control plane - hosts: k8s_control +# - name: configure control plane +# hosts: k8s_control +# gather_facts: true +# become: true +# roles: +# - k8s_control + +- name: configure calico networking + hosts: localhost gather_facts: true - become: true + become: false roles: - - k8s_control + - k8s_network diff --git a/ansible/playbooks/88_scratch.yaml b/ansible/playbooks/88_scratch.yaml index 630d646..e02fd85 100644 --- a/ansible/playbooks/88_scratch.yaml +++ b/ansible/playbooks/88_scratch.yaml @@ -1,7 +1,7 @@ --- - name: scratch testing - hosts: localhost - gather_facts: false - become: false + hosts: k8s_control + gather_facts: true + become: true roles: - scratch \ No newline at end of file diff --git a/ansible/roles/k8s_control/defaults/main.yaml b/ansible/roles/k8s_control/defaults/main.yaml index 3480aa9..ea2c497 100644 --- a/ansible/roles/k8s_control/defaults/main.yaml +++ b/ansible/roles/k8s_control/defaults/main.yaml @@ -1,6 +1,7 @@ --- k8s_control_master_node: kube01.balsillie.net k8s_endpoint: k8s.balsillie.net +k8s_address: "192.168.199.240" k8s_api_port: "6443" k8s_cri_socket: /run/containerd/containerd.sock k8s_service_domain: cluster.internal diff --git a/ansible/roles/k8s_control/tasks/main.yaml b/ansible/roles/k8s_control/tasks/main.yaml index 4b0d301..02695cb 100644 --- a/ansible/roles/k8s_control/tasks/main.yaml +++ b/ansible/roles/k8s_control/tasks/main.yaml @@ -1,4 +1,10 @@ --- +- name: write cluster api address to hosts file + ansible.builtin.lineinfile: + line: "{{ k8s_address }} {{ k8s_endpoint }}" + insertafter: EOF + path: /etc/hosts + - name: create containerd config dir ansible.builtin.file: path: /etc/containerd @@ -53,6 +59,7 @@ --node-name {{ ansible_hostname }} \ --feature-gates IPv6DualStack=false \ --feature-gates PublicKeysECDSA=true \ + --skip-phases=addon/kube-proxy \ --service-dns-domain {{ k8s_service_domain }} \ creates: /etc/kubernetes/admin.conf register: k8s_init @@ -102,5 +109,6 @@ --control-plane \ --certificate-key {{ hostvars[groups['k8s_control'][0]]['join_key']['stdout_lines'][2] }} \ --cri-socket /run/containerd/containerd.sock \ + --skip-phases=addon/kube-proxy \ --node-name {{ ansible_hostname }} creates: /etc/kubernetes/admin.conf diff --git a/ansible/roles/k8s_network/defaults/main.yaml b/ansible/roles/k8s_network/defaults/main.yaml new file mode 100644 index 0000000..91ba9f5 --- /dev/null +++ b/ansible/roles/k8s_network/defaults/main.yaml @@ -0,0 +1,20 @@ +--- +calico_version: v3.24.3 +k8s_network_packages: + - kubernetes-tools + - kubectl-plugins + - python-kubernetes + - python-yaml + - python-jsonpatch +k8s_pod_cidr: 10.128.0.0/16 +k8s_network_blocksize: 20 +k8s_network_encapsulation: None +k8s_network_nat: Disabled +k8s_network_bgp: Enabled +k8s_network_dataplane: BPF +k8s_network_hostports: Disabled +k8s_network_bgp_peer_name: opnsense +k8s_network_bgp_peer_address: 192.168.199.254 +k8s_network_bgp_peer_as: 64612 +k8s_endpoint: k8s.balsillie.net +k8s_api_port: "6443" \ No newline at end of file diff --git a/ansible/roles/k8s_network/tasks/main.yaml b/ansible/roles/k8s_network/tasks/main.yaml new file mode 100644 index 0000000..74c835b --- /dev/null +++ b/ansible/roles/k8s_network/tasks/main.yaml @@ -0,0 +1,119 @@ +--- +- name: ensure required python bindings are present + when: ansible_os_family == 'Archlinux' + become: true + community.general.pacman: + name: "{{ k8s_network_packages }}" + state: latest + update_cache: true + +- name: create target directory for calico files + ansible.builtin.file: + path: "{{ ansible_search_path[0] }}/files/calico" + state: directory + mode: 0775 + +- name: download the calico operator manifest + ansible.builtin.uri: + url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/tigera-operator.yaml + dest: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml" + creates: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml" + mode: 0664 + +- name: download calico configuration + ansible.builtin.uri: + url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/custom-resources.yaml + dest: "{{ ansible_search_path[0] }}/files/calico/calico_resources_{{ calico_version }}.yaml" + creates: "{{ ansible_search_path[0] }}/files/calico/calico_resources_{{ calico_version }}.yaml" + mode: 0664 + +- name: read the default config into memory + ansible.builtin.slurp: + src: "{{ ansible_search_path[0] }}/files/calico/calico_resources_{{ calico_version }}.yaml" + register: calico_file_raw + +- name: split and parse calico settings from the file data # to_yaml will reject the --- in the original manifest, hence data must be split. + ansible.builtin.set_fact: + calico_default_installation: "{{ (calico_file_raw['content'] | b64decode).split(\"---\")[0] | from_yaml }}" + calico_default_apiserver: "{{ (calico_file_raw['content'] | b64decode).split(\"---\")[1] | from_yaml }}" + +- name: update calico installation settings to desired values + ansible.utils.update_fact: + updates: + - path: calico_default_installation.spec.calicoNetwork.ipPools[0].blockSize + value: "{{ k8s_network_blocksize }}" + - path: calico_default_installation.spec.calicoNetwork.ipPools[0].cidr + value: "{{ k8s_pod_cidr }}" + - path: calico_default_installation.spec.calicoNetwork.ipPools[0].encapsulation + value: "{{ k8s_network_encapsulation }}" + - path: calico_default_installation.spec.calicoNetwork.ipPools[0].natOutgoing + value: "{{ k8s_network_nat }}" + - path: calico_default_installation.spec.calicoNetwork.bgp + value: "{{ k8s_network_bgp }}" + - path: calico_default_installation.spec.calicoNetwork.linuxDataplane + value: "{{ k8s_network_dataplane }}" + - path: calico_default_installation.spec.calicoNetwork.hostPorts + value: "{{ k8s_network_hostports }}" + register: calico_updated_installation + +- name: add config map for ebpf mode # https://projectcalico.docs.tigera.io/maintenance/ebpf/install + ansible.builtin.set_fact: + calico_configmap_ebpf: + kind: ConfigMap + apiVersion: v1 + metadata: + name: kubernetes-services-endpoint + namespace: tigera-operator + data: + KUBERNETES_SERVICE_HOST: "{{ k8s_endpoint }}" + KUBERNETES_SERVICE_PORT: "{{ k8s_api_port }}" + +- name: add bgp peer for gateway/router + ansible.builtin.set_fact: + calico_bgp_peer: + apiVersion: crd.projectcalico.org/v1 + kind: BGPPeer + metadata: + name: "{{ k8s_network_bgp_peer_name }}" + spec: + peerIP: "{{ k8s_network_bgp_peer_address }}" + asNumber: "{{ k8s_network_bgp_peer_as }}" + +- name: write out calico configmap for ebpf mode + ansible.builtin.copy: + content: "{{ calico_configmap_ebpf | to_nice_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element + dest: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml" + +- name: write out calico installation definition + ansible.builtin.copy: + content: "{{ calico_updated_installation.calico_default_installation | to_nice_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element + dest: "{{ ansible_search_path[0] }}/files/calico/calico_installation.yaml" + +- name: write out calico apiserver definition + ansible.builtin.copy: + content: "{{ calico_default_apiserver | to_nice_yaml }}" + dest: "{{ ansible_search_path[0] }}/files/calico/calico_apiserver.yaml" + +- name: write out calico bgp peer definition + ansible.builtin.copy: + content: "{{ calico_bgp_peer | to_nice_yaml }}" + dest: "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml" + +- name: install configmap for ebpf mode to cluster + kubernetes.core.k8s: + src: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml" + state: present + +- name: install calico operator to cluster + kubernetes.core.k8s: + src: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml" + state: present + +- name: install calico definitions to cluster + kubernetes.core.k8s: + state: present + src: "{{ item }}" + with_items: + - "{{ ansible_search_path[0] }}/files/calico/calico_installation.yaml" + - "{{ ansible_search_path[0] }}/files/calico/calico_apiserver.yaml" + - "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml" diff --git a/ansible/roles/scratch/tasks/main.yaml b/ansible/roles/scratch/tasks/main.yaml index bf657cb..c2c0c8c 100644 --- a/ansible/roles/scratch/tasks/main.yaml +++ b/ansible/roles/scratch/tasks/main.yaml @@ -1,8 +1,6 @@ --- - name: debug hostname - ansible.builtin.debug: - msg: - - "{{ hostvars[groups['k8s_control'][0]]['ansible_host'] }}" - - "{{ ansible_host }}" - - # ['hosts'][0] \ No newline at end of file + ansible.builtin.lineinfile: + line: 192.168.199.240 k8s.balsillie.net + insertafter: EOF + path: /etc/hosts \ No newline at end of file