#!/bin/bash # Set array variables OSDS=(0 1 2 3 4 5 6 7 8 9 10 11 12 13 14) DATABASE_LIST=( "nextcloud" "gitea" "mastodon-db" "synapse" "mas" "mss" "coturn" "keycloak" "signal-bridge" "whatsapp-bridge" "telegram-bridge" "discord-bridge" "facebook-bridge" ) # Uncordon node echo "Uncordoning node." kubectl uncordon kube00 # Scale bind to 1 echo "Scaling up Bind." kubectl -n dns scale deployment bind --replicas=1 # Wait for the ceph monitors and managers to be ready echo "Waiting for ceph monitors and managers to be ready..." kubectl wait --for=condition=available=True deployment/rook-ceph-mon-a --timeout=30m -n rook-ceph kubectl wait --for=condition=available=True deployment/rook-ceph-mgr-a --timeout=30m -n rook-ceph # Wait for ceph block pool OSDs to be ready echo "Waiting for ceph block pool OSDs to be ready..." kubectl wait --for=condition=available=True deployment/rook-ceph-osd-10 --timeout=30m -n rook-ceph kubectl wait --for=condition=available=True deployment/rook-ceph-osd-11 --timeout=30m -n rook-ceph # Scale up the non-psql db workloads echo "Scaling up non-psql db workloads." kubectl -n db scale deployment --all --replicas=1 # Take all databases out of hibernation echo "Un-hibernate DB 1/13..." kubectl cnpg hibernate off gitea -n db echo "Un-hibernate DB 2/13..." kubectl cnpg hibernate off keycloak -n db echo "Un-hibernate DB 3/13..." kubectl cnpg hibernate off mastodon-db -n db echo "Un-hibernate DB 4/13..." kubectl cnpg hibernate off nextcloud -n db echo "Un-hibernate DB 5/13..." kubectl cnpg hibernate off synapse -n db echo "Un-hibernate DB 6/13..." kubectl cnpg hibernate off mss -n db echo "Un-hibernate DB 7/13..." kubectl cnpg hibernate off mas -n db echo "Un-hibernate DB 8/13..." kubectl cnpg hibernate off coturn -n db echo "Un-hibernate DB 9/13..." kubectl cnpg hibernate off signal-bridge -n db echo "Un-hibernate DB 10/13..." kubectl cnpg hibernate off whatsapp-bridge -n db echo "Un-hibernate DB 11/13..." kubectl cnpg hibernate off telegram-bridge -n db echo "Un-hibernate DB 12/13..." kubectl cnpg hibernate off discord-bridge -n db echo "Un-hibernate DB 13/13..." kubectl cnpg hibernate off facebook-bridge -n db # Scale up Keycloak echo "Scaling up Keycloak." kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/keycloak --timeout=15m -n db kubectl -n public scale statefulset keycloak --replicas=1 kubectl -n public scale deployment keycloak-operator --replicas=1 # Wait for the ceph-fs metadata servers to be ready echo "Waiting for ceph-fs metadata servers to be ready..." kubectl wait --for=condition=available=True deployment/mds-ceph-fs-hdd-a --timeout=30m -n rook-ceph kubectl wait --for=condition=available=True deployment/mds-ceph-fs-hdd-b --timeout=30m -n rook-ceph # Wait for all remaining ceph osds to be ready echo "Waiting for all remaining ceph osds to be ready..." for OSD in "${OSDS[@]}"; do echo "Waiting for OSD $OSD to be ready..." kubectl wait --for=condition=available=True deployment/rook-ceph-osd-$OSD --timeout=30m -n rook-ceph done # Scale up Mail echo "Scaling up Mail." kubectl -n public scale deployment postfix dovecot --replicas=1 # Scale up Nextcloud echo "Scaling up Nextcloud." kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/nextcloud --timeout=15m -n db kubectl -n private scale deployment sftp --replicas=1 kubectl -n public scale deployment nextcloud collabora --replicas=1 # Scale up Gitea echo "Scaling up Gitea." kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/gitea --timeout=15m -n db kubectl -n public scale deployment gitea --replicas=1 # Scale up Mastodon echo "Scaling up Mastodon." kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' cluster/mastodon-db --timeout=15m -n db kubectl -n public scale deployment mastodon --replicas=1 # Scale up all other deployments echo "Scaling up all other deployments." kubectl -n public scale deployment --all --replicas=1 kubectl -n private scale deployment --all --replicas=1 # Notify scale up complete /scripts/notify.sh "Operations resumed on host $HOSTNAME."