open ebs zfs

.vscode/settings.json

@@ -2,5 +2,25 @@
     "yaml.schemas": {
         "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible.json": "file:///home/michael/Code/home/IaC/ansible/roles/vm_deploy/tasks/deploy.yaml",
         "kubernetes://schema/storage.k8s.io/v1@storageclass": "file:///home/michael/Code/home/IaC/ansible/roles/k8s_storage_deploy/files/config/blockpool_ssd_replica.yaml"
+    },
+    "vs-kubernetes": {
+
+        "vs-kubernetes.namespace": "",
+        "disable-linters": ["resource-limits"],
+        "vs-kubernetes.kubectl-path": "",
+        "vs-kubernetes.helm-path": "",
+        "vs-kubernetes.minikube-path": "",
+        "vs-kubernetes.kubectlVersioning": "user-provided",
+        "vs-kubernetes.outputFormat": "yaml",
+        "vs-kubernetes.kubeconfig": "",
+        "vs-kubernetes.knownKubeconfigs": [],
+        "vs-kubernetes.autoCleanupOnDebugTerminate": false,
+        "vs-kubernetes.nodejs-autodetect-remote-root": true,
+        "vs-kubernetes.nodejs-remote-root": "",
+        "vs-kubernetes.nodejs-debug-port": 9229,
+        "vs-kubernetes.dotnet-vsdbg-path": "~/vsdbg/vsdbg",
+        "vs-kubernetes.local-tunnel-debug-provider": "",
+        "checkForMinikubeUpgrade": true,
+        "imageBuildTool": "Docker"
     }
 }

ansible/ansible.cfg

@@ -1,5 +1,5 @@
 [defaults]
-inventory = ./inventory/inventory.yaml
+inventory = ./inventory/
 jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n
 library = modules
 module_utils = module_utils

ansible/inventory/group_vars/all/all.yaml

@@ -7,20 +7,20 @@ ansible_become_method: sudo
 ansible_become_user: root
 ansible_port: 22
 zfs_packages:
+  - linux-lts-headers
+  - openssl-1.1
   - zfs-utils
   - zfs-dkms
-  - mbuffer
   - smartmontools
-  - linux-lts-headers
 zfs_key_id: DDF7DB817396A49B2A2723F7403BD972F75D9D76
 zfs_pools:
   - name: ssd
-    ashift: 13
+    ashift: 16
+    recordsize: 64k
     type: ""
     disks: /dev/vde
+    compression: 'off'
     datasets:
-      - name: ssd
-        encrypt: false
       - name: ssd/data
         encrypt: false
       - name: ssd/data/open-ebs

ansible/inventory/group_vars/k8s_storage/k8s_storage.yaml

@@ -0,0 +1,2 @@
+---
+

ansible/inventory/host_vars/localhost/localhost.yaml

@@ -0,0 +1,3 @@
+ebs_zfs_storage_classes:
+  - name: ssd-zfs
+    dataset: ssd/data/open-ebs

ansible/playbooks/05_k8s_deploy.yaml

@@ -20,19 +20,33 @@
 #   roles:
 #     - k8s_taint
 
-# - name: configure ebs storage operator
+# - name: configure zfs storage on nodes
+#   hosts: k8s_storage
+#   gather_facts: true
+#   become: true
+#   roles:
+#     - zfs_repo_install
+
+# - name: configure open-ebs storage operator
 #   hosts: localhost
 #   gather_facts: false
 #   become: false
 #   roles:
-#     - k8s_storage_ebs_deploy
+#     - k8s_storage_ebs_local_deploy
 
-- name: configure smb storage provider
+- name: configure open-ebs zfs driver
   hosts: localhost
   gather_facts: false
   become: false
   roles:
-    - k8s_storage_smb_deploy
+    - k8s_storage_ebs_zfs_deploy
+
+# - name: configure smb storage provider
+#   hosts: localhost
+#   gather_facts: false
+#   become: false
+#   roles:
+#     - k8s_storage_smb_deploy
 
 # - name: configure ingress controller
 #   hosts: localhost

ansible/roles/k8s_storage_ebs_local_deploy/templates/ebs_storage_class.yaml.j2

@@ -10,6 +10,6 @@ metadata:
         value: "hostpath"
       - name: BasePath
         value: "/ebs/{{ item }}/"
-volumeBindingMode: Immediate
 allowVolumeExpansion: true
+volumeBindingMode: WaitForFirstConsumer
 reclaimPolicy: Retain

ansible/roles/k8s_storage_ebs_zfs_deploy/files/sc_ssd-zfs.yaml

@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ssd-zfs
+parameters:
+  poolname: ssd/data/open-ebs
+  fstype: zfs
+provisioner: zfs.csi.openebs.io
+allowVolumeExpansion: true
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Retain

ansible/roles/k8s_storage_ebs_zfs_deploy/tasks/main.yaml

@@ -0,0 +1,23 @@
+---
+- name: download zfs operator manifest
+  ansible.builtin.uri:
+    url: https://openebs.github.io/charts/zfs-operator.yaml
+    dest: "{{ ansible_search_path[0] }}/files/zfs-operator.yaml"
+    creates: "{{ ansible_search_path[0] }}/files/zfs-operator.yaml"
+
+- name: install zfs operator to cluster
+  kubernetes.core.k8s:
+    src: "{{ ansible_search_path[0] }}/files/zfs-operator.yaml"
+    state: present
+
+- name: template out the storage classes
+  ansible.builtin.template:
+    src: sc-zfs.yaml.j2
+    dest: "{{ ansible_search_path[0] }}/files/sc_{{ item.name }}.yaml"
+  loop: "{{ ebs_zfs_storage_classes }}"
+
+- name: install zfs storage classes to cluster
+  kubernetes.core.k8s:
+    src: "{{ ansible_search_path[0] }}/files/sc_{{ item.name }}.yaml"
+    state: present
+  loop: "{{ ebs_zfs_storage_classes }}"

ansible/roles/k8s_storage_ebs_zfs_deploy/templates/sc-zfs.yaml.j2

@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ item.name }}
+parameters:
+  poolname: {{ item.dataset }}
+  fstype: zfs
+provisioner: zfs.csi.openebs.io
+allowVolumeExpansion: true
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Retain

ansible/roles/zfs_repo_install/tasks/main.yml

@@ -23,7 +23,7 @@
     ansible.builtin.uri:
       dest: "/tmp/{{ zfs_key_id }}"
       url: https://archzfs.com/archzfs.gpg
-    creates: "/tmp/{{ zfs_key_id }}"
+      creates: "/tmp/{{ zfs_key_id }}"
 
   - name: install and lsign zfs key
     ansible.builtin.shell:
@@ -41,34 +41,35 @@
     community.general.pacman:
       name: "{{ zfs_packages }}"
       state: latest
-
-  - name: set zfs module parameters
-    ansible.builtin.template:
-      src: zfs.conf.j2
-      dest: /etc/modprobe.d/zfs.conf
-      owner: root
-      group: root
-      mode: 0664
-
-  - name: load zfs module
-    community.general.modprobe:
-      name: zfs
-      state: present
-
-  - name: enable zfs services
-    become: true
-    ansible.builtin.service:
-      name: "{{ item }}"
-      state: started
-      enabled: yes
-    loop:
-      - zfs-import-cache.service
-      - zfs-mount.service
-      - zfs.target
+    register: zfs_installed
 
   - name: reboot system
     ansible.builtin.reboot:
       post_reboot_delay: 90
+    when: zfs_installed is changed
+
+- name: set zfs module parameters
+  ansible.builtin.template:
+    src: zfs.conf.j2
+    dest: /etc/modprobe.d/zfs.conf
+    owner: root
+    group: root
+    mode: 0664
+
+- name: load zfs module
+  community.general.modprobe:
+    name: zfs
+    state: present
+
+- name: enable zfs services
+  ansible.builtin.service:
+    name: "{{ item }}"
+    state: started
+    enabled: yes
+  loop:
+    - zfs-import-cache.service
+    - zfs-mount.service
+    - zfs.target
 
 - name: gather existing zpool facts
   community.general.zpool_facts: 
@@ -97,19 +98,14 @@
         -O canmount=off \
         -O devices=off \
         -O compression={{ item.compression }} \
+        -O reservation=none \
+        -O refreservation=none \
+        -O recordsize={{ item.recordsize }} \
         {{ item.name }} {{ item.type }} {{ item.disks }} 
   when: item.name not in ( ansible_zfs_pools | selectattr("name") | list )
   with_items:
     "{{ zfs_pools }}"
 
-- name: gather existing zfs dataset facts
-  community.general.zfs_facts:
-    name: "{{ item.name }}"
-    recurse: true
-    type: filesystem
-  with_items:
-    "{{ zfs_pools }}"
-
 - name: create zfs datasets
   loop: "{{ zfs_pools | subelements('datasets') }}"
   community.general.zfs:
@@ -118,16 +114,16 @@
     extra_zfs_properties:
       canmount: off
       mountpoint: none
-      primarycache: none
+      primarycache: metadata
       secondarycache: none
       reservation: none
       refreservation: none
       dedup: off
       encryption: off
+      compression: off
       volmode: dev
       devices: off
       atime: off
-  when: item.1.name not in ( ansible_zfs_datasets | selectattr("name") | list )
   
 
 

notes/2023-01-09_ebs-zfs.txt

@@ -1,7 +1,7 @@
 created a zfs zvol on hv00 ssd
 sudo zfs create -V 100G \
     -o compression=off \
-    -o volblocksize=16k \
+    -o volblocksize=64k \
     -o reservation=none \
     -o refreservation=none \
     -o primarycache=metadata \