Add k8s shutdown/openup scripts

Add nut ansible roles
Add acme certificate ansible role

scripts/notify.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+curl -d "$1" -X POST https://$NOTIFY_HOST/$NOTIFY_CHANNEL

scripts/openup.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+
+# Set array variables
+
+OSDS=(0 1 2 3 4 5 6 7 8 9 10 11 12 13 14)
+DATABASE_LIST=(
+    "nextcloud"
+    "gitea"
+    "mastodon-db"
+    "synapse"
+    "mas"
+    "mss"
+    "coturn"
+    "keycloak"
+    "signal-bridge"
+    "whatsapp-bridge"
+    "telegram-bridge"
+    "discord-bridge"
+    "facebook-bridge"
+)
+
+# Uncordon node
+
+echo "Uncordoning node."
+kubectl uncordon kube00
+
+# Scale bind to 1
+
+echo "Scaling up Bind."
+kubectl -n dns scale deployment bind --replicas=1
+
+# Wait for the ceph monitors and managers to be ready
+
+echo "Waiting for ceph monitors and managers to be ready..."
+kubectl wait --for=condition=available=True deployment/rook-ceph-mon-a --timeout=30m -n rook-ceph
+kubectl wait --for=condition=available=True deployment/rook-ceph-mgr-a --timeout=30m -n rook-ceph
+
+# Wait for ceph block pool OSDs to be ready
+
+echo "Waiting for ceph block pool OSDs to be ready..."
+kubectl wait --for=condition=available=True deployment/rook-ceph-osd-10 --timeout=30m -n rook-ceph
+kubectl wait --for=condition=available=True deployment/rook-ceph-osd-11 --timeout=30m -n rook-ceph
+
+# Scale up the non-psql db workloads
+
+echo "Scaling up non-psql db workloads."
+kubectl -n db scale deployment --all --replicas=1
+
+# Take all databases out of hibernation
+
+echo "Un-hibernate DB 1/13..."
+kubectl cnpg hibernate off gitea -n db
+echo "Un-hibernate DB 2/13..."
+kubectl cnpg hibernate off keycloak -n db
+echo "Un-hibernate DB 3/13..."
+kubectl cnpg hibernate off mastodon-db -n db
+echo "Un-hibernate DB 4/13..."
+kubectl cnpg hibernate off nextcloud -n db
+echo "Un-hibernate DB 5/13..."
+kubectl cnpg hibernate off synapse -n db
+echo "Un-hibernate DB 6/13..."
+kubectl cnpg hibernate off mss -n db
+echo "Un-hibernate DB 7/13..."
+kubectl cnpg hibernate off mas -n db
+echo "Un-hibernate DB 8/13..."
+kubectl cnpg hibernate off coturn -n db
+echo "Un-hibernate DB 9/13..."
+kubectl cnpg hibernate off signal-bridge -n db
+echo "Un-hibernate DB 10/13..."
+kubectl cnpg hibernate off whatsapp-bridge -n db
+echo "Un-hibernate DB 11/13..."
+kubectl cnpg hibernate off telegram-bridge -n db
+echo "Un-hibernate DB 12/13..."
+kubectl cnpg hibernate off discord-bridge -n db
+echo "Un-hibernate DB 13/13..."
+kubectl cnpg hibernate off facebook-bridge -n db
+
+# Scale up Keycloak
+
+echo "Scaling up Keycloak."
+kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/keycloak --timeout=15m -n db
+kubectl -n public scale statefulset keycloak --replicas=1
+kubectl -n public scale deployment keycloak-operator --replicas=1
+
+# Wait for the ceph-fs metadata servers to be ready
+
+echo "Waiting for ceph-fs metadata servers to be ready..."
+kubectl wait --for=condition=available=True deployment/mds-ceph-fs-hdd-a --timeout=30m -n rook-ceph
+kubectl wait --for=condition=available=True deployment/mds-ceph-fs-hdd-b --timeout=30m -n rook-ceph
+
+# Wait for all remaining ceph osds to be ready
+
+echo "Waiting for all remaining ceph osds to be ready..."
+for OSD in "${OSDS[@]}"; do
+    echo "Waiting for OSD $OSD to be ready..."
+    kubectl wait --for=condition=available=True deployment/rook-ceph-osd-$OSD --timeout=30m -n rook-ceph
+done
+
+# Scale up Mail
+
+echo "Scaling up Mail."
+kubectl -n public scale deployment postfix dovecot --replicas=1
+
+# Scale up Nextcloud
+
+echo "Scaling up Nextcloud."
+kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/nextcloud --timeout=15m -n db
+kubectl -n private scale deployment sftp --replicas=1
+kubectl -n public scale deployment nextcloud collabora --replicas=1
+
+# Scale up Gitea
+
+echo "Scaling up Gitea."
+kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/gitea --timeout=15m -n db
+kubectl -n public scale deployment gitea --replicas=1
+
+# Scale up Mastodon
+
+echo "Scaling up Mastodon."
+kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/mastodon-db --timeout=15m -n db
+kubectl -n public scale deployment mastodon --replicas=1
+
+# Scale up all other deployments
+
+echo "Scaling up all other deployments."
+kubectl -n public scale deployment --all --replicas=1
+kubectl -n private scale deployment --all --replicas=1
+
+# Notify scale up complete
+
+/scripts/notify.sh "Operations resumed on host $HOSTNAME."

scripts/shutdown.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Set array variables
+
+DATABASE_LIST=(
+    "nextcloud"
+    "gitea"
+    "mastodon-db"
+    "synapse"
+    "mas"
+    "mss"
+    "coturn"
+    "keycloak"
+    "signal-bridge"
+    "whatsapp-bridge"
+    "telegram-bridge"
+    "discord-bridge"
+    "facebook-bridge"
+)
+
+# Notify shutdown commencement
+
+/scripts/notify.sh "Shutdown initiated on host $HOSTNAME."
+
+# Scale keycloak first
+
+kubectl -n public scale deployment keycloak-operator --replicas=0
+kubectl -n public scale statefulset keycloak --replicas=0
+
+# Scale all deployments to 0
+
+kubectl -n private scale deployment --all --replicas=0
+kubectl -n public scale deployment --selector=delayed.shutdown!=enabled --replicas=0 # Leave ntfy running
+kubectl -n db scale deployment --all --replicas=0
+
+# Notify scaling complete
+
+/scripts/notify.sh "Application scale down complete."
+
+# Hibernate all databases
+
+DB_INDEX=1
+DB_TOTAL=${#DATABASE_LIST[@]}
+for DB in "${DATABASE_LIST[@]}"; do
+    echo "Hibernating database $DB_INDEX/$DB_TOTAL ($DB)..."
+    kubectl cnpg hibernate on $DB -n db
+    DB_INDEX=$((DB_INDEX+1))
+done
+
+# Notify hibernation complete
+
+/scripts/notify.sh "Database hibernations complete, initiating final shutdown."
+
+# Scale the last deployments (ntfy + dns)
+
+kubectl -n public scale deployment --selector=delayed.shutdown=enabled --replicas=0
+kubectl -n dns scale deployment bind --replicas=0
+
+# Cordon node
+
+kubectl cordon kube00
+
+# Drain remaining pods
+
+kubectl drain kube00 --ignore-daemonsets --delete-local-data
+
+# Shutdown upsmon to notify the UPS primary that secondary shutdown has finished
+
+systemctl stop nut-monitor.service
+
+# shutdown
+
+poweroff