// gpu-turnstile is a GPU arbitration proxy that sits in front of Ollama and // ComfyUI and guarantees only one of them uses the GPU at a time. package main import ( "bufio" "context" "errors" "fmt" "io" "io/fs" "log/slog" "net" "net/http" "os" "os/signal" "path/filepath" "runtime" "strings" "syscall" "time" "gpu-turnstile/internal/comfy" "gpu-turnstile/internal/config" "gpu-turnstile/internal/game" "gpu-turnstile/internal/lock" "gpu-turnstile/internal/metrics" "gpu-turnstile/internal/ollama" "gpu-turnstile/internal/proxy" "gpu-turnstile/internal/service" "gpu-turnstile/internal/supervise" "gpu-turnstile/internal/update" ) // version is injected at build time via -ldflags "-X main.version=...". var version = "dev" // exitCodeUpdate tells the service recovery configuration to restart the // process: a signed update has been staged and the GPU lock is idle. const exitCodeUpdate = 3 // stdoutIsTerminal reports whether stdout is a console (char device), as // opposed to a pipe or file — which is what Docker containers and services // see. func stdoutIsTerminal() bool { fi, err := os.Stdout.Stat() return err == nil && fi.Mode()&os.ModeCharDevice != 0 } // parseFlags extracts -config (or -config=), the // --install-service / --remove-service switches, --no-copy, -h/--help, // -v/--version, --force-update and the hidden --elevated-child marker from // args. func parseFlags(args []string) (configPath string, install, remove, noCopy, help, showVersion, forceUpdate, elevatedChild bool, rest []string) { rest = args[:0] for i := 0; i < len(args); i++ { switch { case args[i] == "-config" && i+1 < len(args): configPath = args[i+1] i++ case strings.HasPrefix(args[i], "-config="): configPath = strings.TrimPrefix(args[i], "-config=") case args[i] == "--install-service" || args[i] == "-install-service": install = true case args[i] == "--remove-service" || args[i] == "-remove-service": remove = true case args[i] == "--no-copy" || args[i] == "-no-copy": noCopy = true case args[i] == "-h" || args[i] == "--help" || args[i] == "-help": help = true case args[i] == "-v" || args[i] == "--version" || args[i] == "-version": showVersion = true case args[i] == "--force-update" || args[i] == "-force-update": forceUpdate = true case args[i] == "--elevated-child": elevatedChild = true default: rest = append(rest, args[i]) } } return configPath, install, remove, noCopy, help, showVersion, forceUpdate, elevatedChild, rest } // versionLine is printed at the top of every help and error screen. func versionLine() string { return "gpu-turnstile " + version } const usageText = `GPU arbitration proxy for Ollama + ComfyUI Usage: gpu-turnstile -config run the proxy gpu-turnstile --install-service [--no-copy] [-config path] install + start as a service gpu-turnstile --remove-service stop + uninstall the service gpu-turnstile -v | --version print just the version gpu-turnstile --force-update check for a signed update now, apply it and restart the service gpu-turnstile -h | --help this help Options: -config config file (default: gpu-turnstile.env next to the exe) --install-service copies the binary into the canonical location (%ProgramFiles%\gpu-turnstile or /var/lib/gpu-turnstile) unless --no-copy; on Windows a UAC prompt appears when the shell is not elevated --no-copy with --install-service: register the current location as-is All runtime settings are environment variables or KEY=VALUE lines in the config file (OLLAMA_URL, COMFY_URL, LOGLEVEL, ...); see README.md. ` // printHelp prints the version header plus the full help text. func printHelp() { fmt.Printf("%s — %s", versionLine(), usageText) } // fatalUsage prints the version header, an error message and the one-line // usage summary, then exits with code 2. func fatalUsage(format string, args ...any) { fmt.Fprintf(os.Stderr, "%s\n\n", versionLine()) fmt.Fprintf(os.Stderr, format+"\n\n", args...) fmt.Fprintln(os.Stderr, "usage: gpu-turnstile [-config path] [--install-service [--no-copy] | --remove-service]") fmt.Fprintln(os.Stderr, " gpu-turnstile service install|remove [-config path]") os.Exit(2) } func main() { configPath, install, remove, noCopy, help, showVersion, forceUpdate, elevatedChild, args := parseFlags(os.Args[1:]) if showVersion { fmt.Println(version) return } bare := configPath == "" && !install && !remove && !forceUpdate && !elevatedChild && len(args) == 0 if help || (bare && stdoutIsTerminal()) { // Bare invocation in a terminal (e.g. double-clicked on Windows) // shows the help instead of starting a proxy window with no visible // explanation. Without a terminal — Docker containers, services, // pipes — a bare invocation starts the proxy as before. printHelp() return } if len(args) > 0 && args[0] == "service" { // Legacy subcommand form: gpu-turnstile service install|remove. if len(args) != 2 || (args[1] != "install" && args[1] != "remove") { fatalUsage("error: expected 'service install' or 'service remove'") } install = args[1] == "install" remove = !install args = nil } switch { case install && remove: fatalUsage("error: --install-service and --remove-service are mutually exclusive") case forceUpdate && (install || remove): fatalUsage("error: --force-update cannot be combined with --install-service/--remove-service") case install: os.Exit(serviceCommand(configPath, true, noCopy, elevatedChild)) case remove: os.Exit(serviceCommand(configPath, false, noCopy, elevatedChild)) case forceUpdate: os.Exit(forceUpdateCommand(configPath, elevatedChild)) } if len(args) > 0 { fatalUsage("error: unknown arguments: %s", strings.Join(args, " ")) } if exePath, err := os.Executable(); err == nil { update.CleanupOld(exePath) } cfg, err := loadMergedConfig(configPath) if err != nil { fmt.Fprintf(os.Stderr, "%s\n\ngpu-turnstile: %v\n", versionLine(), err) os.Exit(1) } log, logOut, logCloser := newLogger(cfg) defer logCloser.Close() syncEnvFile(resolveConfigPath(configPath), cfg.LogFile, log) if service.IsService() { if err := service.Run(func(ctx context.Context) error { return run(ctx, cfg, log, logOut, true) }); err != nil { log.Error("service failed", "err", err) os.Exit(1) } return } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() if err := run(ctx, cfg, log, logOut, false); err != nil { log.Error("listener failed", "err", err) os.Exit(1) } } // syncEnvFile upgrades an installer-written config file after an update: // settings added since its CFG_VER are appended (commented out) and CFG_VER // is bumped. Files not written by the installer (no CFG_VER), up-to-date // files and dev builds are left untouched; a write failure is logged, not // fatal. func syncEnvFile(path, logFile string, log *slog.Logger) { data, err := os.ReadFile(path) if err != nil { return // no config file; nothing to upgrade } synced, changed := config.SyncSample(string(data), version, logFile) if !changed { return } if err := os.WriteFile(path, []byte(synced), 0o644); err != nil { log.Warn("could not append new settings to the config file", "path", path, "err", err) return } log.Warn("config file updated: new settings appended", "path", path, "version", version) } // defaultConfigPath returns gpu-turnstile.env next to the executable. func defaultConfigPath() string { exe, err := os.Executable() if err != nil { return "gpu-turnstile.env" } return filepath.Join(filepath.Dir(exe), "gpu-turnstile.env") } // resolveConfigPath applies the precedence: -config flag, then // GPU_TURNSTILE_CONFIG, then the default next to the executable. func resolveConfigPath(flagValue string) string { if flagValue != "" { return flagValue } if v := os.Getenv("GPU_TURNSTILE_CONFIG"); v != "" { return v } return defaultConfigPath() } // loadMergedConfig reads the config file (if present) and overlays process // environment variables on top. A missing file is fine; an unreadable or // malformed file is fatal. func loadMergedConfig(flagValue string) (config.Config, error) { path := resolveConfigPath(flagValue) values := map[string]string{} if f, err := os.Open(path); err == nil { defer f.Close() parsed, err := config.ParseEnvFile(f) if err != nil { return config.Config{}, fmt.Errorf("%s: %w", path, err) } values = parsed } else if !errors.Is(err, os.ErrNotExist) { return config.Config{}, fmt.Errorf("read config file: %w", err) } getenv := func(key string) string { if v := os.Getenv(key); v != "" { return v } return values[key] } return config.Load(getenv) } // newLogger builds the slog logger and returns the output writer (stderr or // the opened LOG_FILE) plus a closer for it. func newLogger(cfg config.Config) (*slog.Logger, io.Writer, io.Closer) { out := io.Writer(os.Stderr) closer := io.NopCloser(nil) if cfg.LogFile != "" { if f, err := os.OpenFile(cfg.LogFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644); err == nil { out = f closer = f } else { fmt.Fprintf(os.Stderr, "gpu-turnstile: cannot open LOG_FILE %s: %v (logging to stderr)\n", cfg.LogFile, err) } } opts := &slog.HandlerOptions{Level: cfg.LogLevel} var handler slog.Handler = slog.NewTextHandler(out, opts) if cfg.LogJSON { handler = slog.NewJSONHandler(out, opts) } log := slog.New(handler) slog.SetDefault(log) return log, out, closer } // waitForEnter keeps an elevated child's console window open until the // user has read the output. func waitForEnter() { fmt.Print("\nPress Enter to close this window...") bufio.NewReader(os.Stdin).ReadString('\n') } // elevateAndMirror relaunches the current command elevated (UAC) and // mirrors the child's exit code. verb is used in messages. func elevateAndMirror(verb string) (int, bool) { args := append(append([]string{}, os.Args[1:]...), "--elevated-child") code, err := service.RelaunchElevated(args) if errors.Is(err, service.ErrUserCancelled) { fmt.Fprintln(os.Stderr, "gpu-turnstile: UAC prompt declined") return 1, true } if err != nil { fmt.Fprintf(os.Stderr, "gpu-turnstile: could not elevate: %v\n", err) return 1, true } if code != 0 { fmt.Fprintf(os.Stderr, "gpu-turnstile %s failed in the elevated process (exit %d)\n", verb, code) return code, true } return 0, true } // isPermission reports whether err is a permission problem (Windows // ERROR_ACCESS_DENIED, POSIX EACCES/EPERM, possibly wrapped). func isPermission(err error) bool { return errors.Is(err, fs.ErrPermission) || strings.Contains(strings.ToLower(err.Error()), "access is denied") } // forceUpdateCommand checks for a signed update immediately, stages it if // newer, and restarts the service when it is running so the new binary // takes effect. Staging into a system directory and restarting a service // need admin rights; instead of prompting unconditionally, permission // failures trigger the UAC relaunch so a dev copy in a user-writable // directory updates without a prompt. func forceUpdateCommand(configPath string, elevatedChild bool) int { if elevatedChild { defer waitForEnter() } cfg, err := loadMergedConfig(configPath) if errors.Is(err, config.ErrNoConsumer) { err = nil // update settings do not depend on a consumer URL } if err != nil { fmt.Fprintf(os.Stderr, "%s\n\ngpu-turnstile: %v\n", versionLine(), err) return 1 } exePath, err := os.Executable() if err != nil { fmt.Fprintf(os.Stderr, "gpu-turnstile: cannot locate executable: %v\n", err) return 1 } log, _, logCloser := newLogger(cfg) defer logCloser.Close() if cfg.AppVersion == "dev" { fmt.Printf("%s: APP_VER=dev, updates disabled\n", versionLine()) return 0 } u := &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log} // Single-shot: one attempt, fail fast when the server is unreachable // instead of hanging in a TCP connect for minutes. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() staged, err := u.Check(ctx, exePath) if err != nil && isPermission(err) && !service.Elevated() { code, _ := elevateAndMirror("--force-update") if code == 0 { fmt.Println("update applied (elevated)") } return code } if err != nil { fmt.Fprintf(os.Stderr, "%s\n\ngpu-turnstile: update check failed: %v\n", versionLine(), err) return 1 } if !staged { fmt.Printf("%s is up to date\n", versionLine()) return 0 } fmt.Printf("%s: update staged\n", versionLine()) restarted, err := service.RestartIfRunning() if err != nil && isPermission(err) && !service.Elevated() { code, _ := elevateAndMirror("--force-update") if code == 0 { fmt.Println("update applied (elevated)") } return code } if err != nil { fmt.Fprintf(os.Stderr, "gpu-turnstile: update staged but service restart failed: %v\n", err) return 1 } if restarted { fmt.Println("service restarted on the new version") } else { fmt.Println("no running service; the new version applies on next start") } return 0 } // serviceCommand installs (copyBin = register the canonical-layout copy) // or removes the service and reports the result. On Windows, when the // shell is not elevated, the command relaunches itself through a UAC // prompt and mirrors the elevated child's exit code. An elevated child // waits for a keypress so its console window does not flash closed before // the output can be read. func serviceCommand(configPath string, install, noCopy, elevatedChild bool) int { verb, doneVerb := "remove", "removed" if install { verb, doneVerb = "install", "installed" } if elevatedChild { defer waitForEnter() } if !service.Elevated() { code, done := elevateAndMirror(verb) if done && code != 0 { return code } if done { fmt.Printf("service %s: %s (elevated)\n", service.Name, doneVerb) return 0 } } var err error if install { path := resolveConfigPath(configPath) if abs, absErr := filepath.Abs(path); absErr == nil { path = abs } err = service.Install(path, !noCopy, version) } else { err = service.Remove() } if err != nil { fmt.Fprintf(os.Stderr, "gpu-turnstile service %s: %v\n", verb, err) return 1 } fmt.Printf("service %s: %s\n", service.Name, doneVerb) return 0 } // orDisabled renders an empty URL as "disabled" for the startup dump. func orDisabled(url string) string { if url == "" { return "disabled" } return url } // managedComfyCommand resolves how ComfyUI is launched when it is managed: // COMFY_CMD verbatim, or the standard venv layout under COMFY_DIR. Empty // when neither is set (unmanaged). func managedComfyCommand(cfg config.Config) string { if cfg.ComfyCmd != "" { return cfg.ComfyCmd } if cfg.ComfyDir != "" { return supervise.DefaultComfyCommand(runtime.GOOS, cfg.ComfyDir, cfg.ComfyURL) } return "" } func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Writer, isService bool) error { // ComfyUI can run as a managed child — COMFY_CMD verbatim, or the // standard venv layout derived from COMFY_DIR alone: started on demand // by the proxy, stopped after COMFY_IDLE_TIMEOUT idle (and on shutdown) // so its VRAM is freed. comfyCmdLine := managedComfyCommand(cfg) // The startup line carries the version and every setting and is emitted // at WARN so it is visible even with the default (quiet) log level. log.Log(ctx, slog.LevelWarn, "starting gpu-turnstile", "version", version, "listen_ollama", cfg.ListenOllama, "listen_comfy", cfg.ListenComfy, "ollama_url", orDisabled(cfg.OllamaURL), "comfy_url", orDisabled(cfg.ComfyURL), "unload_timeout", cfg.UnloadTimeout, "job_timeout", cfg.JobTimeout, "llm_wait_timeout", cfg.LLMWaitTimeout, "llm_busy_mode", cfg.LLMBusyMode, "llm_busy_status", cfg.LLMBusyStatus, "busy_retry_after", cfg.BusyRetryAfter, "unload_poll_interval", cfg.UnloadPollInterval, "history_poll_interval", cfg.HistoryPollInterval, "probe_timeout", cfg.ProbeTimeout, "health_interval", cfg.HealthInterval, "free_timeout", cfg.FreeTimeout, "warm_timeout", cfg.WarmTimeout, "shutdown_timeout", cfg.ShutdownTimeout, "backoff_initial", cfg.BackoffInitial, "backoff_max", cfg.BackoffMax, "prompt_capture_limit", cfg.PromptCaptureLimit, "warm_model", cfg.WarmModel, "comfy_cmd", orDisabled(comfyCmdLine), "comfy_dir", cfg.ComfyDir, "comfy_idle_timeout", cfg.ComfyIdleTimeout, "comfy_start_timeout", cfg.ComfyStartTimeout, "game_procs", cfg.GameProcs, "gpu_foreign_vram_mb", cfg.GPUForeignVRAMMB, "gpu_ignore_procs", cfg.GPUIgnoreProcs, "game_poll_interval", cfg.GamePollInterval, "auto_update", cfg.AutoUpdate, "update_interval", cfg.UpdateInterval, "update_repo", cfg.UpdateRepo, "update_asset", cfg.UpdateAsset, "log_level", cfg.LogLevel, "log_format", map[bool]string{true: "json", false: "text"}[cfg.LogJSON], "log_file", cfg.LogFile, ) lk := lock.New(log) // Each consumer is enabled by setting its URL; a disabled consumer gets // no client, no listener and no probe. var ollamaClient *ollama.Client var err error if cfg.OllamaURL != "" { if ollamaClient, err = ollama.New(cfg.OllamaURL, log); err != nil { return err } } var comfyClient *comfy.Client if cfg.ComfyURL != "" { if comfyClient, err = comfy.New(cfg.ComfyURL, log); err != nil { return err } } var comfySup *supervise.Process if comfyCmdLine != "" { if cfg.ComfyCmd == "" { // Derived from COMFY_DIR: flag a wrong-looking layout early, // while the operator is still watching the startup log. python, script := supervise.ComfyLayout(runtime.GOOS, cfg.ComfyDir) for _, p := range []string{python, script} { if _, err := os.Stat(p); err != nil { log.Warn("COMFY_DIR: file not found; ComfyUI requests will fail until it exists", "path", p) } } } var err error comfySup, err = supervise.New("comfy", comfyCmdLine, cfg.ComfyDir, comfyClient.Probe, cfg.ComfyStartTimeout, log) if err != nil { return err } defer comfySup.Stop() gpuIdle := func() bool { state, _, pending := lk.Snapshot() return state == lock.StateIdle && !pending } go comfySup.WatchIdle(ctx, cfg.ComfyIdleTimeout, gpuIdle) } srv, err := proxy.New(proxy.Config{ OllamaURL: cfg.OllamaURL, ComfyURL: cfg.ComfyURL, Lock: lk, Ollama: ollamaClient, Comfy: comfyClient, ComfySup: comfySup, Metrics: metrics.New(), Log: log, LogColor: !cfg.LogJSON && cfg.LogFile == "" && os.Getenv("NO_COLOR") == "", LogWriter: logOut, LLMWaitTimeout: cfg.LLMWaitTimeout, UnloadTimeout: cfg.UnloadTimeout, JobTimeout: cfg.JobTimeout, UnloadPollInterval: cfg.UnloadPollInterval, HistoryPollInterval: cfg.HistoryPollInterval, FreeTimeout: cfg.FreeTimeout, WarmTimeout: cfg.WarmTimeout, LLMBusyMode: cfg.LLMBusyMode, LLMBusyStatus: cfg.LLMBusyStatus, BusyRetryAfter: cfg.BusyRetryAfter, BackoffInitial: cfg.BackoffInitial, BackoffMax: cfg.BackoffMax, PromptCaptureLimit: cfg.PromptCaptureLimit, WarmModel: cfg.WarmModel, }) if err != nil { return err } // Probe the enabled upstreams once; failure is logged, not fatal. A // managed ComfyUI is intentionally down at startup — the first request // starts it — so neither the startup probe nor the health check treats // that as an outage. probes := map[string]func(context.Context) error{} if ollamaClient != nil { probes["ollama"] = ollamaClient.Probe } if comfyClient != nil { if comfySup == nil { probes["comfy"] = comfyClient.Probe } else { // Managed upstream: an idle-stopped or still-starting server is // not an outage, so it is skipped until it has answered once // (Ready resets on every spawn/stop). After that, a failed // probe while the process lives is a real "DOWN". probes["comfy"] = func(ctx context.Context) error { if !comfySup.Ready() { if comfySup.Running() { if err := comfyClient.Probe(ctx); err == nil { comfySup.MarkReady() } } return errManagedDown } return comfyClient.Probe(ctx) } } } probeCtx, probeCancel := context.WithTimeout(ctx, cfg.ProbeTimeout) for name, probe := range probes { if err := probe(probeCtx); err != nil && !errors.Is(err, errManagedDown) { log.Warn(name+" probe failed", "err", err) } } probeCancel() if cfg.HealthInterval > 0 { go healthLoop(ctx, cfg.HealthInterval, cfg.ProbeTimeout, log, probes) } // Foreign GPU holders (games, other ML jobs) — enabled by GAME_PROCS // and/or GPU_FOREIGN_VRAM_MB — hold the lock externally while they run. if len(cfg.GameProcs) > 0 || cfg.GPUForeignVRAMMB > 0 { det := game.New(cfg.GameProcs, cfg.GPUForeignVRAMMB, cfg.GPUIgnoreProcs, log) go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup) } // Bind the listeners up front so a port conflict fails fast and the // readiness notification below really means "accepting connections". var servers []*http.Server var listeners []net.Listener bind := func(addr string, handler http.Handler, consumer string) error { ln, err := net.Listen("tcp", addr) if err != nil { return fmt.Errorf("listen %s on %s: %w", consumer, addr, err) } servers = append(servers, &http.Server{Addr: addr, Handler: handler}) listeners = append(listeners, ln) log.Warn("listening", "consumer", consumer, "addr", addr) return nil } if ollamaClient != nil { if err := bind(cfg.ListenOllama, srv.OllamaHandler(), "ollama"); err != nil { return err } } if comfyClient != nil { if err := bind(cfg.ListenComfy, srv.ComfyHandler(), "comfy"); err != nil { return err } } errCh := make(chan error, len(servers)) for i := range servers { go func(s *http.Server, ln net.Listener) { errCh <- s.Serve(ln) }(servers[i], listeners[i]) } // Tell systemd we are up and start the watchdog pings; both are no-ops // when not running under a notify/watchdog unit. service.NotifyReady() service.StartWatchdog(ctx) if cfg.AutoUpdate { go updateLoop(ctx, cfg, log, lk, isService) } select { case err := <-errCh: if err != nil && !errors.Is(err, http.ErrServerClosed) { return err } case <-ctx.Done(): log.Info("shutting down") } shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout) defer shutdownCancel() for _, s := range servers { s.Shutdown(shutdownCtx) } return nil } // errManagedDown marks a managed upstream that is intentionally stopped // (idle); health checks skip it instead of logging an outage. var errManagedDown = errors.New("managed upstream intentionally stopped") // gameLoop polls for foreign GPU holders (a game, another ML job). While one // is detected it holds the lock externally so new LLM and image requests // wait (or are rejected per LLM_BUSY_MODE), and — once in-flight work has // drained — frees VRAM for it: the managed ComfyUI is stopped and Ollama's // resident models are unloaded. func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process) { ticker := time.NewTicker(cfg.GamePollInterval) defer ticker.Stop() held, freed := false, false for { select { case <-ctx.Done(): return case <-ticker.C: } holders, err := det.Check(ctx) if err != nil && ctx.Err() == nil { log.Warn("game detection failed", "err", err) } switch { case len(holders) > 0 && !held: held = true lk.SetExternal(summarizeHolders(holders)) log.Warn("GPU held by an external process; new LLM/image requests wait", "holders", summarizeHolders(holders)) case len(holders) == 0 && held: held, freed = false, false lk.ClearExternal() log.Warn("external process released the GPU; resuming") } if held && !freed { if state, _, _ := lk.Snapshot(); state != lock.StateLLM && state != lock.StateImage { freed = true freeVRAM(ctx, cfg.UnloadTimeout, log, ollamaClient, comfySup) } } } } // summarizeHolders joins holder descriptions for logs and busy responses, // capping the list so a process name matching dozens of PIDs (system // services) does not flood the log. func summarizeHolders(holders []string) string { const max = 5 if len(holders) > max { return strings.Join(holders[:max], "; ") + fmt.Sprintf("; +%d more", len(holders)-max) } return strings.Join(holders, "; ") } // freeVRAM stops the managed ComfyUI (never an external server on its port) // and unloads Ollama's resident models so the foreign process gets the GPU // memory. func freeVRAM(ctx context.Context, unloadTimeout time.Duration, log *slog.Logger, ollamaClient *ollama.Client, comfySup *supervise.Process) { if comfySup != nil && comfySup.Running() { log.Warn("stopping the managed ComfyUI to free VRAM") comfySup.Stop() } if ollamaClient == nil { return } uctx, cancel := context.WithTimeout(ctx, unloadTimeout) defer cancel() if models, err := ollamaClient.LoadedModels(uctx); err != nil || len(models) == 0 { return // nothing resident (or ollama unreachable); nothing to free } elapsed, err := ollamaClient.UnloadAll(uctx) if err != nil { log.Warn("ollama unload incomplete; continuing", "err", err) return } log.Warn("ollama models unloaded to free VRAM", "seconds", elapsed.Seconds()) } // healthLoop probes the enabled upstreams every interval and logs status // transitions — "is DOWN" when a previously healthy upstream stops // answering, "recovered" when it comes back. The first round only // establishes the baseline; the startup probe already reported that state. func healthLoop(ctx context.Context, interval, probeTimeout time.Duration, log *slog.Logger, probes map[string]func(context.Context) error) { ticker := time.NewTicker(interval) defer ticker.Stop() up := map[string]bool{} managed := map[string]bool{} for { select { case <-ctx.Done(): return case <-ticker.C: } for name, probe := range probes { pctx, cancel := context.WithTimeout(ctx, probeTimeout) err := probe(pctx) cancel() if errors.Is(err, errManagedDown) { managed[name] = true // intentionally stopped; not an outage continue } now := err == nil if managed[name] { // First real probe after an idle stop only re-baselines — // an on-demand start is not a "recovery". managed[name] = false up[name] = now continue } was, seen := up[name] if seen && now != was { if now { log.Warn(name + " upstream recovered") } else { log.Warn(name+" upstream is DOWN", "err", err) } } up[name] = now } } } // updateLoop checks for signed updates on startup and every UPDATE_INTERVAL. // In service mode a staged update is applied by exiting with exitCodeUpdate // once the GPU lock is idle; the service recovery configuration restarts the // process with the new binary. Interactively it only logs. func updateLoop(ctx context.Context, cfg config.Config, log *slog.Logger, lk *lock.Lock, isService bool) { exePath, err := os.Executable() if err != nil { log.Warn("auto-update disabled: cannot locate executable", "err", err) return } u := &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log} for { staged, err := u.Check(ctx, exePath) if err != nil && ctx.Err() == nil { log.Warn("auto-update check failed", "err", err) } if staged { if !isService { log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply") return } log.Warn("auto-update: staged; restarting once the GPU is idle") if waitForIdle(ctx, lk, 24*time.Hour) { log.Warn("auto-update: restarting to apply update") os.Exit(exitCodeUpdate) } return } select { case <-ctx.Done(): return case <-time.After(cfg.UpdateInterval): } } } // waitForIdle polls the lock until no LLM or image work is active or // pending, max at most. Returns false on timeout or cancellation. func waitForIdle(ctx context.Context, lk *lock.Lock, max time.Duration) bool { deadline := time.Now().Add(max) for { if state, _, pending := lk.Snapshot(); state == lock.StateIdle && !pending { return true } if time.Now().After(deadline) { return false } select { case <-ctx.Done(): return false case <-time.After(5 * time.Second): } } }