Periodic upstream health checks (HEALTH_INTERVAL, default 30s); log down/recovered transitions

cmd/gpu-turnstile/main.go

@@ -428,6 +428,7 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
 		"unload_poll_interval", cfg.UnloadPollInterval,
 		"history_poll_interval", cfg.HistoryPollInterval,
 		"probe_timeout", cfg.ProbeTimeout,
+		"health_interval", cfg.HealthInterval,
 		"free_timeout", cfg.FreeTimeout,
 		"warm_timeout", cfg.WarmTimeout,
 		"shutdown_timeout", cfg.ShutdownTimeout,
@@ -491,18 +492,23 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
 	}
 
 	// Probe the enabled upstreams once; failure is logged, not fatal.
-	probeCtx, probeCancel := context.WithTimeout(ctx, cfg.ProbeTimeout)
+	probes := map[string]func(context.Context) error{}
 	if ollamaClient != nil {
-		if err := ollamaClient.Probe(probeCtx); err != nil {
-			log.Warn("ollama probe failed", "url", cfg.OllamaURL, "err", err)
-		}
+		probes["ollama"] = ollamaClient.Probe
 	}
 	if comfyClient != nil {
-		if err := comfyClient.Probe(probeCtx); err != nil {
-			log.Warn("comfy probe failed", "url", cfg.ComfyURL, "err", err)
+		probes["comfy"] = comfyClient.Probe
+	}
+	probeCtx, probeCancel := context.WithTimeout(ctx, cfg.ProbeTimeout)
+	for name, probe := range probes {
+		if err := probe(probeCtx); err != nil {
+			log.Warn(name+" probe failed", "err", err)
 		}
 	}
 	probeCancel()
+	if cfg.HealthInterval > 0 {
+		go healthLoop(ctx, cfg.HealthInterval, cfg.ProbeTimeout, log, probes)
+	}
 
 	// Bind the listeners up front so a port conflict fails fast and the
 	// readiness notification below really means "accepting connections".
@@ -560,6 +566,38 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
 	return nil
 }
 
+// healthLoop probes the enabled upstreams every interval and logs status
+// transitions — "is DOWN" when a previously healthy upstream stops
+// answering, "recovered" when it comes back. The first round only
+// establishes the baseline; the startup probe already reported that state.
+func healthLoop(ctx context.Context, interval, probeTimeout time.Duration, log *slog.Logger, probes map[string]func(context.Context) error) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	up := map[string]bool{}
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+		for name, probe := range probes {
+			pctx, cancel := context.WithTimeout(ctx, probeTimeout)
+			err := probe(pctx)
+			cancel()
+			was, seen := up[name]
+			now := err == nil
+			if seen && now != was {
+				if now {
+					log.Warn(name + " upstream recovered")
+				} else {
+					log.Warn(name+" upstream is DOWN", "err", err)
+				}
+			}
+			up[name] = now
+		}
+	}
+}
+
 // updateLoop checks for signed updates on startup and every UPDATE_INTERVAL.
 // In service mode a staged update is applied by exiting with exitCodeUpdate
 // once the GPU lock is idle; the service recovery configuration restarts the