From 8792112b4722f9f088ba67bd61fa8c08ccebb1aa Mon Sep 17 00:00:00 2001
From: Negash <i@negash.ru>
Date: Wed, 29 Jul 2026 22:13:30 +0200
Subject: [PATCH] fix: watchdog DNS tolerance (3 consecutive failures) +
 pg_isready in wait-service
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Watchdog was too aggressive — single nslookup failure (transient CoreDNS
hiccup, network blip) triggered tunnel kill and unnecessary re-provisioning.
Now requires 3 consecutive DNS failures before killing the tunnel.

wait-service was checking only TCP port 5432 (nc -z), not PostgreSQL
readiness. PostgreSQL from snapshot needs WAL recovery time — TCP port
opens before PostgreSQL accepts connections. Now uses pg_isready to wait
for actual PostgreSQL readiness, not just TCP port.
---
 chart/templates/generator-application.yaml |  8 ++++++--
 tunnel/aldaas.sh                           | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/chart/templates/generator-application.yaml b/chart/templates/generator-application.yaml
index 5d5fdd7..2aced62 100644
--- a/chart/templates/generator-application.yaml
+++ b/chart/templates/generator-application.yaml
@@ -34,11 +34,15 @@ spec:
     - name: wait-service
       container:
         args:
-          - while ! nc -z '{{`{{workflow.name}}`}}' {{ .Values.application.port }}; do sleep 1; done
+          - >-
+            while ! nc -z '{{`{{workflow.name}}`}}' {{ .Values.application.port }}; do sleep 1; done;
+            echo "TCP port ready, waiting for PostgreSQL to accept connections...";
+            until PGPASSWORD={{ .Values.application.password | default "passw0rd" }} pg_isready -h '{{`{{workflow.name}}`}}' -p {{ .Values.application.port }} -U {{ .Values.application.user | default "app" }} -d {{ .Values.application.db | default "app" }}; do sleep 2; done;
+            echo "PostgreSQL is ready to accept connections"
         command:
           - sh
           - -c
-        image: alpine
+        image: postgres:17-alpine
     - name: get-last-snapshot
       container:
         name: main
diff --git a/tunnel/aldaas.sh b/tunnel/aldaas.sh
index e322b4b..9498c79 100755
--- a/tunnel/aldaas.sh
+++ b/tunnel/aldaas.sh
@@ -110,13 +110,23 @@ while true; do
     # Watchdog: monitor upstream Service DNS
     # When the Argo workflow is deleted (TTL expiry / remove EventSource),
     # the Service DNS record is removed. We detect this and trigger re-provisioning.
+    # Tolerance: require N consecutive DNS failures before killing the tunnel,
+    # to avoid false positives from transient CoreDNS hiccups or network blips.
+    DNS_FAIL_COUNT=0
+    DNS_FAIL_THRESHOLD=3
     while kill -0 $TUNNEL_PID 2>/dev/null; do
         sleep 10
         if ! nslookup "$aldaas_name.$ALDAAS_NAMESPACE.svc.cluster.local" >/dev/null 2>&1; then
-            echo "Upstream Service DNS gone — re-provisioning"
-            kill $TUNNEL_PID 2>/dev/null
-            wait $TUNNEL_PID 2>/dev/null
-            break
+            DNS_FAIL_COUNT=$((DNS_FAIL_COUNT + 1))
+            echo "DNS lookup failed ($DNS_FAIL_COUNT/$DNS_FAIL_THRESHOLD) for $aldaas_name.$ALDAAS_NAMESPACE.svc.cluster.local"
+            if [ "$DNS_FAIL_COUNT" -ge "$DNS_FAIL_THRESHOLD" ]; then
+                echo "Upstream Service DNS gone after $DNS_FAIL_THRESHOLD consecutive failures — re-provisioning"
+                kill $TUNNEL_PID 2>/dev/null
+                wait $TUNNEL_PID 2>/dev/null
+                break
+            fi
+        else
+            DNS_FAIL_COUNT=0
         fi
     done