Periodic upstream health checks (HEALTH_INTERVAL, default 30s); log down/recovered transitions

internal/config/config.go

@@ -26,6 +26,7 @@ type Config struct {
 	UnloadPollInterval  time.Duration
 	HistoryPollInterval time.Duration
 	ProbeTimeout        time.Duration
+	HealthInterval      time.Duration
 	FreeTimeout         time.Duration
 	WarmTimeout         time.Duration
 	ShutdownTimeout     time.Duration
@@ -70,6 +71,7 @@ func Defaults() Config {
 		UnloadPollInterval:  500 * time.Millisecond,
 		HistoryPollInterval: time.Second,
 		ProbeTimeout:        5 * time.Second,
+		HealthInterval:      30 * time.Second,
 		FreeTimeout:         30 * time.Second,
 		WarmTimeout:         2 * time.Minute,
 		ShutdownTimeout:     10 * time.Second,
@@ -166,6 +168,7 @@ func Load(getenv func(string) string) (Config, error) {
 		{"UNLOAD_POLL_INTERVAL", &cfg.UnloadPollInterval},
 		{"HISTORY_POLL_INTERVAL", &cfg.HistoryPollInterval},
 		{"PROBE_TIMEOUT", &cfg.ProbeTimeout},
+		{"HEALTH_INTERVAL", &cfg.HealthInterval},
 		{"FREE_TIMEOUT", &cfg.FreeTimeout},
 		{"WARM_TIMEOUT", &cfg.WarmTimeout},
 		{"SHUTDOWN_TIMEOUT", &cfg.ShutdownTimeout},

internal/config/sample.go

@@ -39,7 +39,8 @@ func sampleEntries(logFile string) []sampleEntry {
 		{"LOG_FILE", logFile, "Append logs to this file instead of stderr (a Windows service has no console)", logFile != ""},
 		{"UNLOAD_POLL_INTERVAL", "500ms", "/api/ps poll interval while unloading", false},
 		{"HISTORY_POLL_INTERVAL", "1s", "/history/<id> poll interval while a job runs", false},
-		{"PROBE_TIMEOUT", "5s", "Startup probe timeout for the enabled upstreams", false},
+		{"PROBE_TIMEOUT", "5s", "Probe timeout for the startup probe and the periodic health check", false},
+		{"HEALTH_INTERVAL", "30s", "How often enabled upstreams are probed; status changes are logged", false},
 		{"FREE_TIMEOUT", "30s", "Timeout for the POST /free call after an image job", false},
 		{"WARM_TIMEOUT", "2m", "Timeout for the warm-model reload after an image job", false},
 		{"SHUTDOWN_TIMEOUT", "10s", "Graceful shutdown timeout on SIGINT/SIGTERM", false},