PUBLIC/gpu-turnstile
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Game detection: foreign GPU holders take an external lock hold (GAME_PROCS, GPU_FOREIGN_VRAM_MB)

Browse Source
This commit is contained in:
mram
2026-09-21 17:16:56 +02:00
parent 14120bf4a4
commit e4bdc92ece
16 changed files with 736 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/proxy/proxy.go
+33 -6
View File
@@ -464,10 +464,14 @@ func (s *Server) OllamaHandler() http.Handler {
if s.busyMode == "reject" {
if !s.cfg.Lock.TryAcquireLLM() {
s.cfg.Metrics.ObserveLockWait("llm", time.Since(start).Seconds())
msg := "GPU busy: image job active or queued"
if holder := s.cfg.Lock.External(); holder != "" {
msg = "GPU busy: " + holder
}
s.log.Info("llm request rejected; GPU busy",
"path", r.URL.Path, "status", s.busyStatus)
w.Header().Set("Retry-After", strconv.Itoa(s.busyRetryAfter))
http.Error(w, "GPU busy: image job active or queued", s.busyStatus)
http.Error(w, msg, s.busyStatus)
return
}
s.cfg.Metrics.ObserveLockWait("llm", time.Since(start).Seconds())
@@ -508,7 +512,14 @@ func (s *Server) ComfyHandler() http.Handler {
}
if s.cfg.ComfySup != nil {
// Any other ComfyUI request also wakes the managed server; the
// retry backoff bridges the time it needs to come up.
// retry backoff bridges the time it needs to come up. While a
// foreign process holds the GPU we refuse to spawn it — the
// request gets the busy answer instead of fighting for VRAM.
if holder := s.cfg.Lock.External(); holder != "" && !s.cfg.ComfySup.Running() {
w.Header().Set("Retry-After", strconv.Itoa(s.busyRetryAfter))
http.Error(w, "GPU busy: "+holder, http.StatusServiceUnavailable)
return
}
if err := s.cfg.ComfySup.EnsureRunning(); err != nil {
http.Error(w, fmt.Sprintf("cannot start ComfyUI: %v", err), http.StatusBadGateway)
return
@@ -553,10 +564,13 @@ func (w *captureWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
func (s *Server) handlePrompt(w http.ResponseWriter, r *http.Request) {
log := s.log.With("op", "image")
if s.cfg.ComfySup != nil {
// Bring the managed server up *before* taking the GPU lock: torch
// can take a minute to load, and LLM traffic should keep flowing
// in the meantime.
// Normally the managed server is brought up *before* taking the GPU
// lock: torch can take a minute to load, and LLM traffic should keep
// flowing in the meantime. While a foreign process holds the GPU, LLM
// traffic is blocked anyway and a fresh ComfyUI would fight it for
// VRAM — so the lock comes first in that case.
comfyFirst := s.cfg.ComfySup != nil && s.cfg.Lock.External() == ""
if comfyFirst {
if err := s.cfg.ComfySup.EnsureRunning(); err != nil {
http.Error(w, fmt.Sprintf("cannot start ComfyUI: %v", err), http.StatusBadGateway)
return
@@ -577,6 +591,19 @@ func (s *Server) handlePrompt(w http.ResponseWriter, r *http.Request) {
s.cfg.Metrics.ObserveLockWait("image", time.Since(start).Seconds())
log.Info("image lock acquired")
if s.cfg.ComfySup != nil && !comfyFirst {
if err := s.cfg.ComfySup.EnsureRunning(); err != nil {
s.cfg.Lock.ReleaseImage()
http.Error(w, fmt.Sprintf("cannot start ComfyUI: %v", err), http.StatusBadGateway)
return
}
if err := s.cfg.ComfySup.WaitReady(r.Context()); err != nil {
s.cfg.Lock.ReleaseImage()
http.Error(w, fmt.Sprintf("ComfyUI did not become ready: %v", err), http.StatusBadGateway)
return
}
}
if s.cfg.Ollama != nil {
uctx, ucancel := context.WithTimeout(r.Context(), s.cfg.UnloadTimeout)
elapsed, uerr := s.cfg.Ollama.UnloadAll(uctx)