Supervisor probes before spawning: an external server on the port is used, never fought or killed

This commit is contained in:
mram
2026-09-21 16:06:28 +02:00
parent d7566329ae
commit 14120bf4a4
4 changed files with 64 additions and 2 deletions
+6
View File
@@ -112,6 +112,12 @@ COMFY_DIR=C:\ComfyUI
(LLM traffic keeps flowing while torch loads); crashes are logged and the (LLM traffic keeps flowing while torch loads); crashes are logged and the
next request respawns. Shutting gpu-turnstile down stops the child too. next request respawns. Shutting gpu-turnstile down stops the child too.
Running the ComfyUI Desktop app alongside is safe: if something already
answers on the port, gpu-turnstile just uses it instead of spawning
(and never kills it — it only ever stops its own child). If the managed
instance already holds the port when you open the desktop app, the
desktop's server is the one that fails to bind.
## Build and run ## Build and run
```sh ```sh
+6
View File
@@ -138,6 +138,12 @@ process instead of expecting an always-on server:
freeing the VRAM ComfyUI holds. The next request restarts it. freeing the VRAM ComfyUI holds. The next request restarts it.
- **Crash**: an unexpected exit is logged; the next request respawns. - **Crash**: an unexpected exit is logged; the next request respawns.
gpu-turnstile's own shutdown stops the child too. gpu-turnstile's own shutdown stops the child too.
- **Coexistence**: before spawning, the URL is probed — if another server
already answers (e.g. the ComfyUI desktop app), it is used as-is and no
child is spawned; the idle watcher and shutdown only ever stop the
supervisor's own process, never the external one. The other direction —
starting the desktop app while the managed instance holds the port —
makes the *desktop* server fail to bind; gpu-turnstile is unaffected.
- Its stdout/stderr is forwarded to the log at INFO. The health check - Its stdout/stderr is forwarded to the log at INFO. The health check
skips the intentionally-stopped/starting states; a failed probe while skips the intentionally-stopped/starting states; a failed probe while
the process is alive and was previously ready is logged as DOWN. the process is alive and was previously ready is logged as DOWN.
+17 -1
View File
@@ -29,6 +29,7 @@ type Process struct {
cmd *exec.Cmd cmd *exec.Cmd
stopping bool stopping bool
ready bool ready bool
external bool // someone else serves the port; not our process
lastActivity time.Time lastActivity time.Time
} }
@@ -118,7 +119,10 @@ func (p *Process) NoteActivity() {
// EnsureRunning starts the child if it is not running. It returns as soon // EnsureRunning starts the child if it is not running. It returns as soon
// as the process is spawned; readiness is WaitReady's job (and the proxy's // as the process is spawned; readiness is WaitReady's job (and the proxy's
// retry backoff bridges the gap for plain proxied requests). // retry backoff bridges the gap for plain proxied requests). When the URL
// already answers — e.g. the ComfyUI desktop app grabbed the port — no
// child is spawned: the external server is used as-is, and the idle
// watcher never touches it (it only kills its own child).
func (p *Process) EnsureRunning() error { func (p *Process) EnsureRunning() error {
p.mu.Lock() p.mu.Lock()
defer p.mu.Unlock() defer p.mu.Unlock()
@@ -126,6 +130,18 @@ func (p *Process) EnsureRunning() error {
if p.cmd != nil { if p.cmd != nil {
return nil return nil
} }
pctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
err := p.probe(pctx)
cancel()
if err == nil {
p.ready = true
if !p.external {
p.external = true
p.log.Info(p.name + " is already served externally; not spawning a managed instance")
}
return nil
}
p.external = false
cmd := exec.Command(p.argv[0], p.argv[1:]...) cmd := exec.Command(p.argv[0], p.argv[1:]...)
cmd.Dir = p.dir cmd.Dir = p.dir
stdout, err := cmd.StdoutPipe() stdout, err := cmd.StdoutPipe()
+35 -1
View File
@@ -57,8 +57,10 @@ func TestHelperProcess(t *testing.T) {
func newHelper(t *testing.T, name string) *Process { func newHelper(t *testing.T, name string) *Process {
t.Helper() t.Helper()
// The probe always fails: nothing external serves the port, so
// EnsureRunning spawns the helper child.
p, err := New(name, `"`+os.Args[0]+`" -test.run=TestHelperProcess`, "", p, err := New(name, `"`+os.Args[0]+`" -test.run=TestHelperProcess`, "",
func(context.Context) error { return nil }, 5*time.Second, slog.Default()) func(context.Context) error { return errors.New("nothing there") }, 5*time.Second, slog.Default())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -103,6 +105,38 @@ func TestEnsureRunningAndStop(t *testing.T) {
waitStopped(t, p, 5*time.Second) waitStopped(t, p, 5*time.Second)
} }
func TestEnsureRunningPrefersExternalServer(t *testing.T) {
// The port is already served (e.g. the ComfyUI desktop app): no child
// is spawned, the supervisor reports ready, and Stop is a no-op.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
probe := func(ctx context.Context) error {
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
resp.Body.Close()
return nil
}
p, err := New("external", `"`+os.Args[0]+`"`, "", probe, 5*time.Second, slog.Default())
if err != nil {
t.Fatal(err)
}
if err := p.EnsureRunning(); err != nil {
t.Fatal(err)
}
if p.Running() {
t.Fatal("spawned a child even though the port is already served")
}
if !p.Ready() {
t.Fatal("external server should count as ready")
}
p.Stop() // must not touch the external server
}
func TestWaitReady(t *testing.T) { func TestWaitReady(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)