Supervisor probes before spawning: an external server on the port is used, never fought or killed
This commit is contained in:
@@ -112,6 +112,12 @@ COMFY_DIR=C:\ComfyUI
|
|||||||
(LLM traffic keeps flowing while torch loads); crashes are logged and the
|
(LLM traffic keeps flowing while torch loads); crashes are logged and the
|
||||||
next request respawns. Shutting gpu-turnstile down stops the child too.
|
next request respawns. Shutting gpu-turnstile down stops the child too.
|
||||||
|
|
||||||
|
Running the ComfyUI Desktop app alongside is safe: if something already
|
||||||
|
answers on the port, gpu-turnstile just uses it instead of spawning
|
||||||
|
(and never kills it — it only ever stops its own child). If the managed
|
||||||
|
instance already holds the port when you open the desktop app, the
|
||||||
|
desktop's server is the one that fails to bind.
|
||||||
|
|
||||||
## Build and run
|
## Build and run
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|||||||
@@ -138,6 +138,12 @@ process instead of expecting an always-on server:
|
|||||||
freeing the VRAM ComfyUI holds. The next request restarts it.
|
freeing the VRAM ComfyUI holds. The next request restarts it.
|
||||||
- **Crash**: an unexpected exit is logged; the next request respawns.
|
- **Crash**: an unexpected exit is logged; the next request respawns.
|
||||||
gpu-turnstile's own shutdown stops the child too.
|
gpu-turnstile's own shutdown stops the child too.
|
||||||
|
- **Coexistence**: before spawning, the URL is probed — if another server
|
||||||
|
already answers (e.g. the ComfyUI desktop app), it is used as-is and no
|
||||||
|
child is spawned; the idle watcher and shutdown only ever stop the
|
||||||
|
supervisor's own process, never the external one. The other direction —
|
||||||
|
starting the desktop app while the managed instance holds the port —
|
||||||
|
makes the *desktop* server fail to bind; gpu-turnstile is unaffected.
|
||||||
- Its stdout/stderr is forwarded to the log at INFO. The health check
|
- Its stdout/stderr is forwarded to the log at INFO. The health check
|
||||||
skips the intentionally-stopped/starting states; a failed probe while
|
skips the intentionally-stopped/starting states; a failed probe while
|
||||||
the process is alive and was previously ready is logged as DOWN.
|
the process is alive and was previously ready is logged as DOWN.
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ type Process struct {
|
|||||||
cmd *exec.Cmd
|
cmd *exec.Cmd
|
||||||
stopping bool
|
stopping bool
|
||||||
ready bool
|
ready bool
|
||||||
|
external bool // someone else serves the port; not our process
|
||||||
lastActivity time.Time
|
lastActivity time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,7 +119,10 @@ func (p *Process) NoteActivity() {
|
|||||||
|
|
||||||
// EnsureRunning starts the child if it is not running. It returns as soon
|
// EnsureRunning starts the child if it is not running. It returns as soon
|
||||||
// as the process is spawned; readiness is WaitReady's job (and the proxy's
|
// as the process is spawned; readiness is WaitReady's job (and the proxy's
|
||||||
// retry backoff bridges the gap for plain proxied requests).
|
// retry backoff bridges the gap for plain proxied requests). When the URL
|
||||||
|
// already answers — e.g. the ComfyUI desktop app grabbed the port — no
|
||||||
|
// child is spawned: the external server is used as-is, and the idle
|
||||||
|
// watcher never touches it (it only kills its own child).
|
||||||
func (p *Process) EnsureRunning() error {
|
func (p *Process) EnsureRunning() error {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
@@ -126,6 +130,18 @@ func (p *Process) EnsureRunning() error {
|
|||||||
if p.cmd != nil {
|
if p.cmd != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
pctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
err := p.probe(pctx)
|
||||||
|
cancel()
|
||||||
|
if err == nil {
|
||||||
|
p.ready = true
|
||||||
|
if !p.external {
|
||||||
|
p.external = true
|
||||||
|
p.log.Info(p.name + " is already served externally; not spawning a managed instance")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p.external = false
|
||||||
cmd := exec.Command(p.argv[0], p.argv[1:]...)
|
cmd := exec.Command(p.argv[0], p.argv[1:]...)
|
||||||
cmd.Dir = p.dir
|
cmd.Dir = p.dir
|
||||||
stdout, err := cmd.StdoutPipe()
|
stdout, err := cmd.StdoutPipe()
|
||||||
|
|||||||
@@ -57,8 +57,10 @@ func TestHelperProcess(t *testing.T) {
|
|||||||
|
|
||||||
func newHelper(t *testing.T, name string) *Process {
|
func newHelper(t *testing.T, name string) *Process {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
// The probe always fails: nothing external serves the port, so
|
||||||
|
// EnsureRunning spawns the helper child.
|
||||||
p, err := New(name, `"`+os.Args[0]+`" -test.run=TestHelperProcess`, "",
|
p, err := New(name, `"`+os.Args[0]+`" -test.run=TestHelperProcess`, "",
|
||||||
func(context.Context) error { return nil }, 5*time.Second, slog.Default())
|
func(context.Context) error { return errors.New("nothing there") }, 5*time.Second, slog.Default())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -103,6 +105,38 @@ func TestEnsureRunningAndStop(t *testing.T) {
|
|||||||
waitStopped(t, p, 5*time.Second)
|
waitStopped(t, p, 5*time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEnsureRunningPrefersExternalServer(t *testing.T) {
|
||||||
|
// The port is already served (e.g. the ComfyUI desktop app): no child
|
||||||
|
// is spawned, the supervisor reports ready, and Stop is a no-op.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
probe := func(ctx context.Context) error {
|
||||||
|
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil)
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p, err := New("external", `"`+os.Args[0]+`"`, "", probe, 5*time.Second, slog.Default())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := p.EnsureRunning(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Running() {
|
||||||
|
t.Fatal("spawned a child even though the port is already served")
|
||||||
|
}
|
||||||
|
if !p.Ready() {
|
||||||
|
t.Fatal("external server should count as ready")
|
||||||
|
}
|
||||||
|
p.Stop() // must not touch the external server
|
||||||
|
}
|
||||||
|
|
||||||
func TestWaitReady(t *testing.T) {
|
func TestWaitReady(t *testing.T) {
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|||||||
Reference in New Issue
Block a user