Game detection: foreign GPU holders take an external lock hold (GAME_PROCS, GPU_FOREIGN_VRAM_MB)

cmd/gpu-turnstile/main.go

@@ -21,6 +21,7 @@ import (
 
 	"gpu-turnstile/internal/comfy"
 	"gpu-turnstile/internal/config"
+	"gpu-turnstile/internal/game"
 	"gpu-turnstile/internal/lock"
 	"gpu-turnstile/internal/metrics"
 	"gpu-turnstile/internal/ollama"
@@ -441,6 +442,10 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
 		"comfy_dir", cfg.ComfyDir,
 		"comfy_idle_timeout", cfg.ComfyIdleTimeout,
 		"comfy_start_timeout", cfg.ComfyStartTimeout,
+		"game_procs", cfg.GameProcs,
+		"gpu_foreign_vram_mb", cfg.GPUForeignVRAMMB,
+		"gpu_ignore_procs", cfg.GPUIgnoreProcs,
+		"game_poll_interval", cfg.GamePollInterval,
 		"auto_update", cfg.AutoUpdate,
 		"update_interval", cfg.UpdateInterval,
 		"update_repo", cfg.UpdateRepo,
@@ -555,6 +560,13 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
 		go healthLoop(ctx, cfg.HealthInterval, cfg.ProbeTimeout, log, probes)
 	}
 
+	// Foreign GPU holders (games, other ML jobs) — enabled by GAME_PROCS
+	// and/or GPU_FOREIGN_VRAM_MB — hold the lock externally while they run.
+	if len(cfg.GameProcs) > 0 || cfg.GPUForeignVRAMMB > 0 {
+		det := game.New(cfg.GameProcs, cfg.GPUForeignVRAMMB, cfg.GPUIgnoreProcs, log)
+		go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup)
+	}
+
 	// Bind the listeners up front so a port conflict fails fast and the
 	// readiness notification below really means "accepting connections".
 	var servers []*http.Server
@@ -615,6 +627,80 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
 // (idle); health checks skip it instead of logging an outage.
 var errManagedDown = errors.New("managed upstream intentionally stopped")
 
+// gameLoop polls for foreign GPU holders (a game, another ML job). While one
+// is detected it holds the lock externally so new LLM and image requests
+// wait (or are rejected per LLM_BUSY_MODE), and — once in-flight work has
+// drained — frees VRAM for it: the managed ComfyUI is stopped and Ollama's
+// resident models are unloaded.
+func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process) {
+	ticker := time.NewTicker(cfg.GamePollInterval)
+	defer ticker.Stop()
+	held, freed := false, false
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+		holders, err := det.Check(ctx)
+		if err != nil && ctx.Err() == nil {
+			log.Warn("game detection failed", "err", err)
+		}
+		switch {
+		case len(holders) > 0 && !held:
+			held = true
+			lk.SetExternal(summarizeHolders(holders))
+			log.Warn("GPU held by an external process; new LLM/image requests wait",
+				"holders", summarizeHolders(holders))
+		case len(holders) == 0 && held:
+			held, freed = false, false
+			lk.ClearExternal()
+			log.Warn("external process released the GPU; resuming")
+		}
+		if held && !freed {
+			if state, _, _ := lk.Snapshot(); state != lock.StateLLM && state != lock.StateImage {
+				freed = true
+				freeVRAM(ctx, cfg.UnloadTimeout, log, ollamaClient, comfySup)
+			}
+		}
+	}
+}
+
+// summarizeHolders joins holder descriptions for logs and busy responses,
+// capping the list so a process name matching dozens of PIDs (system
+// services) does not flood the log.
+func summarizeHolders(holders []string) string {
+	const max = 5
+	if len(holders) > max {
+		return strings.Join(holders[:max], "; ") + fmt.Sprintf("; +%d more", len(holders)-max)
+	}
+	return strings.Join(holders, "; ")
+}
+
+// freeVRAM stops the managed ComfyUI (never an external server on its port)
+// and unloads Ollama's resident models so the foreign process gets the GPU
+// memory.
+func freeVRAM(ctx context.Context, unloadTimeout time.Duration, log *slog.Logger, ollamaClient *ollama.Client, comfySup *supervise.Process) {
+	if comfySup != nil && comfySup.Running() {
+		log.Warn("stopping the managed ComfyUI to free VRAM")
+		comfySup.Stop()
+	}
+	if ollamaClient == nil {
+		return
+	}
+	uctx, cancel := context.WithTimeout(ctx, unloadTimeout)
+	defer cancel()
+	if models, err := ollamaClient.LoadedModels(uctx); err != nil || len(models) == 0 {
+		return // nothing resident (or ollama unreachable); nothing to free
+	}
+	elapsed, err := ollamaClient.UnloadAll(uctx)
+	if err != nil {
+		log.Warn("ollama unload incomplete; continuing", "err", err)
+		return
+	}
+	log.Warn("ollama models unloaded to free VRAM", "seconds", elapsed.Seconds())
+}
+
 // healthLoop probes the enabled upstreams every interval and logs status
 // transitions — "is DOWN" when a previously healthy upstream stops
 // answering, "recovered" when it comes back. The first round only