From a62240b02a49b948e36113dc5b13f084fe1c4be7 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 3 Nov 2022 03:59:42 +1300 Subject: [PATCH] tuning --- ansible/inventory/inventory.yaml | 12 +- ansible/playbooks/05_k8s_deploy.yaml | 2 +- ansible/roles/k8s_control/tasks/main.yaml | 13 +- ansible/roles/k8s_network/defaults/main.yaml | 7 +- ansible/roles/k8s_network/tasks/main.yaml | 53 +++++---- .../roles/k8s_storage_deploy/tasks/main.yaml | 112 +++++++++--------- 6 files changed, 107 insertions(+), 92 deletions(-) diff --git a/ansible/inventory/inventory.yaml b/ansible/inventory/inventory.yaml index 42cde3c..3e43ffc 100644 --- a/ansible/inventory/inventory.yaml +++ b/ansible/inventory/inventory.yaml @@ -25,18 +25,18 @@ all: k8s_control: hosts: kube01: - kube02: - kube03: + # kube02: + # kube03: k8s_taint: hosts: kube01: - kube02: - kube03: + # kube02: + # kube03: k8s_worker: hosts: kube01: - kube02: - kube03: + # kube02: + # kube03: firewalls: children: fortigate: diff --git a/ansible/playbooks/05_k8s_deploy.yaml b/ansible/playbooks/05_k8s_deploy.yaml index 9fd171e..f9c05e8 100644 --- a/ansible/playbooks/05_k8s_deploy.yaml +++ b/ansible/playbooks/05_k8s_deploy.yaml @@ -25,4 +25,4 @@ gather_facts: false become: false roles: - - k8s_storage \ No newline at end of file + - k8s_storage_deploy \ No newline at end of file diff --git a/ansible/roles/k8s_control/tasks/main.yaml b/ansible/roles/k8s_control/tasks/main.yaml index eafffb2..def1b83 100644 --- a/ansible/roles/k8s_control/tasks/main.yaml +++ b/ansible/roles/k8s_control/tasks/main.yaml @@ -50,20 +50,24 @@ ansible.builtin.shell: cmd: | kubeadm init \ - --control-plane-endpoint {{ k8s_endpoint }} \ + --control-plane-endpoint {{ k8s_address }} \ --cri-socket /run/containerd/containerd.sock \ --pod-network-cidr {{ k8s_pod_cidr }} \ --service-cidr {{ k8s_service_cidr }} \ --apiserver-bind-port {{ k8s_api_port }} \ --apiserver-cert-extra-sans {{ k8s_endpoint }} \ + --apiserver-cert-extra-sans {{ k8s_address }} \ --node-name {{ ansible_hostname }} \ --feature-gates IPv6DualStack=false \ --feature-gates PublicKeysECDSA=true \ - --skip-phases=addon/kube-proxy \ --service-dns-domain {{ k8s_service_domain }} \ creates: /etc/kubernetes/admin.conf register: k8s_init + # --skip-phases=addon/kube-proxy \ + +# TODO a check here to wait until node has finished init + - name: register the control plane certificate key ansible.builtin.shell: cmd: | @@ -109,12 +113,13 @@ --control-plane \ --certificate-key {{ hostvars[groups['k8s_control'][0]]['join_key']['stdout_lines'][2] }} \ --cri-socket /run/containerd/containerd.sock \ - --skip-phases=addon/kube-proxy \ --node-name {{ ansible_hostname }} creates: /etc/kubernetes/admin.conf + # --skip-phases=addon/kube-proxy \ + - name: set kubelet service to enabled at startup ansible.builtin.service: name: kubelet.service - state: running + state: started enabled: true diff --git a/ansible/roles/k8s_network/defaults/main.yaml b/ansible/roles/k8s_network/defaults/main.yaml index 91ba9f5..58eed36 100644 --- a/ansible/roles/k8s_network/defaults/main.yaml +++ b/ansible/roles/k8s_network/defaults/main.yaml @@ -11,10 +11,11 @@ k8s_network_blocksize: 20 k8s_network_encapsulation: None k8s_network_nat: Disabled k8s_network_bgp: Enabled -k8s_network_dataplane: BPF -k8s_network_hostports: Disabled +k8s_network_dataplane: Iptables # BPF or Iptables +k8s_network_hostports: Enabled # Disabled when dataplne is BPF k8s_network_bgp_peer_name: opnsense k8s_network_bgp_peer_address: 192.168.199.254 k8s_network_bgp_peer_as: 64612 k8s_endpoint: k8s.balsillie.net -k8s_api_port: "6443" \ No newline at end of file +k8s_api_port: "6443" +k8s_address: "192.168.199.240" \ No newline at end of file diff --git a/ansible/roles/k8s_network/tasks/main.yaml b/ansible/roles/k8s_network/tasks/main.yaml index 2f97eed..39aad89 100644 --- a/ansible/roles/k8s_network/tasks/main.yaml +++ b/ansible/roles/k8s_network/tasks/main.yaml @@ -56,17 +56,27 @@ value: "{{ k8s_network_hostports }}" register: calico_updated_installation -- name: add config map for ebpf mode # https://projectcalico.docs.tigera.io/maintenance/ebpf/install +- name: create tigera operator namespace ansible.builtin.set_fact: - calico_configmap_ebpf: - kind: ConfigMap + calico_namespace: apiVersion: v1 + kind: Namespace metadata: - name: kubernetes-services-endpoint - namespace: tigera-operator - data: - KUBERNETES_SERVICE_HOST: "{{ k8s_endpoint }}" - KUBERNETES_SERVICE_PORT: "{{ k8s_api_port }}" + name: tigera-operator + labels: + name: tigera-operator + +# - name: add config map for ebpf mode # https://projectcalico.docs.tigera.io/maintenance/ebpf/install +# ansible.builtin.set_fact: +# calico_configmap_ebpf: +# kind: ConfigMap +# apiVersion: v1 +# metadata: +# name: kubernetes-services-endpoint +# namespace: tigera-operator +# data: +# KUBERNETES_SERVICE_HOST: "{{ k8s_address }}" +# KUBERNETES_SERVICE_PORT: "{{ k8s_api_port }}" - name: add bgp peer for gateway/router ansible.builtin.set_fact: @@ -79,10 +89,15 @@ peerIP: "{{ k8s_network_bgp_peer_address }}" asNumber: "{{ k8s_network_bgp_peer_as }}" -- name: write out calico configmap for ebpf mode +- name: write out calico namespace crd ansible.builtin.copy: - content: "{{ calico_configmap_ebpf | to_nice_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element - dest: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml" + content: "{{ calico_namespace | to_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element + dest: "{{ ansible_search_path[0] }}/files/calico/calico_namespace.yaml" + +# - name: write out calico configmap for ebpf mode +# ansible.builtin.copy: +# content: "{{ calico_configmap_ebpf | to_yaml }}" # Ansible registers the original fact name (with new vaule) inside the updated fact, hence the sub element +# dest: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml" - name: write out calico installation definition ansible.builtin.copy: @@ -97,26 +112,20 @@ # TODO two api versions exist for BGP, one only becomes available after calico is online. Do they both work?? # crd.projectcalico.org/v1 # projectcalico.org/v3 + - name: write out calico bgp peer definition ansible.builtin.copy: content: "{{ calico_bgp_peer | to_nice_yaml }}" dest: "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml" -- name: install configmap for ebpf mode to cluster - kubernetes.core.k8s: - src: "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml" - state: present - -- name: install calico operator to cluster - kubernetes.core.k8s: - src: "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml" - state: present - -- name: install calico definitions to cluster +- name: install calico definitions to cluster # The order here matters. namespace > config map > operator > installation kubernetes.core.k8s: state: present src: "{{ item }}" with_items: + # - "{{ ansible_search_path[0] }}/files/calico/calico_namespace.yaml" + # - "{{ ansible_search_path[0] }}/files/calico/calico_configmap_ebpf.yaml" + - "{{ ansible_search_path[0] }}/files/calico/calico_operator_{{ calico_version }}.yaml" - "{{ ansible_search_path[0] }}/files/calico/calico_installation.yaml" - "{{ ansible_search_path[0] }}/files/calico/calico_apiserver.yaml" - "{{ ansible_search_path[0] }}/files/calico/calico_bgp_peer.yaml" diff --git a/ansible/roles/k8s_storage_deploy/tasks/main.yaml b/ansible/roles/k8s_storage_deploy/tasks/main.yaml index 9a66302..4461c8c 100644 --- a/ansible/roles/k8s_storage_deploy/tasks/main.yaml +++ b/ansible/roles/k8s_storage_deploy/tasks/main.yaml @@ -1,58 +1,58 @@ --- -# - name: create target directory for rook files -# ansible.builtin.file: -# path: "{{ ansible_search_path[0] }}/files/rook" -# state: directory -# mode: 0775 +- name: create target directory for rook files + ansible.builtin.file: + path: "{{ ansible_search_path[0] }}/files/rook" + state: directory + mode: 0775 -# - name: load rbd kernel module -# become: true -# delegate_to: "{{ item }}" -# with_items: "{{ groups['k8s_worker'] }}" -# community.general.modprobe: -# name: rbd -# state: present +- name: load rbd kernel module + become: true + delegate_to: "{{ item }}" + with_items: "{{ groups['k8s_worker'] }}" + community.general.modprobe: + name: rbd + state: present -# - name: set rbd kernel module to load at boot -# become: true -# delegate_to: "{{ item }}" -# with_items: "{{ groups['k8s_worker'] }}" -# ansible.builtin.copy: -# dest: /etc/modules-load.d/rbd.conf -# content: rbd -# owner: root -# group: root -# mode: 0660 +- name: set rbd kernel module to load at boot + become: true + delegate_to: "{{ item }}" + with_items: "{{ groups['k8s_worker'] }}" + ansible.builtin.copy: + dest: /etc/modules-load.d/rbd.conf + content: rbd + owner: root + group: root + mode: 0660 -# - name: install lvm2 package -# become: true -# delegate_to: "{{ item }}" -# with_items: "{{ groups['k8s_worker'] }}" -# community.general.pacman: -# name: lvm2 -# state: latest -# update_cache: true +- name: install lvm2 package + become: true + delegate_to: "{{ item }}" + with_items: "{{ groups['k8s_worker'] }}" + community.general.pacman: + name: lvm2 + state: latest + update_cache: true -# - name: download the rook manifests -# ansible.builtin.uri: -# url: https://raw.githubusercontent.com/rook/rook/{{ rook_version }}/deploy/examples/{{ item }}.yaml -# dest: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml" -# creates: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml" -# mode: 0664 -# with_items: -# - crds -# - common -# - operator -# - cluster +- name: download the rook manifests + ansible.builtin.uri: + url: https://raw.githubusercontent.com/rook/rook/{{ rook_version }}/deploy/examples/{{ item }}.yaml + dest: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml" + creates: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml" + mode: 0664 + with_items: + - crds + - common + - operator + - cluster -# - name: deploy the rook manifest # The order of the items is important, crds > common > operator , see https://github.com/rook/rook/blob/v1.10.4/deploy/examples/common.yaml -# kubernetes.core.k8s: -# src: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml" -# state: present -# with_items: -# - crds -# - common -# - operator +- name: deploy the rook manifest # The order of the items is important, crds > common > operator , see https://github.com/rook/rook/blob/v1.10.4/deploy/examples/common.yaml + kubernetes.core.k8s: + src: "{{ ansible_search_path[0] }}/files/rook/rook_{{ item }}_{{ rook_version }}.yaml" + state: present + with_items: + - crds + - common + # - operator # # TODO somehow turn this command: # # kubectl -n rook-ceph get pod -o json | jq '.items[].status.containerStatuses[].ready' @@ -92,10 +92,10 @@ # TODO create a check and wait until cluster is created and running -- name: create the storage providers - kubernetes.core.k8s: - src: "{{ ansible_search_path[0] }}/files/config/{{ item }}" - state: present - with_items: - - blockpool_ssd_replica.yaml - - filesystem_multi.yaml \ No newline at end of file +# - name: create the storage providers +# kubernetes.core.k8s: +# src: "{{ ansible_search_path[0] }}/files/config/{{ item }}" +# state: present +# with_items: +# - blockpool_ssd_replica.yaml +# - filesystem_multi.yaml \ No newline at end of file