Add --monitor: live status view (downstream health, GPU lock, queue) via the control channel

internal/control/control.go

@@ -23,6 +23,9 @@ import (
 // idle) restart onto a signed update immediately.
 const CmdUpdateNow = "update-now"
 
+// CmdStatus asks for a one-line JSON status snapshot (monitor mode).
+const CmdStatus = "status"
+
 // ErrUnavailable means no running service offers the control channel.
 var ErrUnavailable = errors.New("control channel unavailable")
 

internal/lock/lock.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"log/slog"
 	"sync"
+	"time"
 )
 
 // State is the current GPU occupancy state.
@@ -30,10 +31,13 @@ type Lock struct {
 	change chan struct{} // closed and replaced on every state change
 
 	n           int  // LLM requests in flight
+	llmWaiting  int  // LLM requests blocked waiting for the GPU
 	imageActive bool // an image job holds the GPU
 	imageQ      []imageWaiter
 	nextID      uint64
-	external    string // non-empty: a foreign process (e.g. a game) holds the GPU
+	external    string    // non-empty: a foreign process (e.g. a game) holds the GPU
+	detail      string    // what the current holder is doing (best effort)
+	since       time.Time // when the current state began
 
 	log *slog.Logger
 }
@@ -41,7 +45,7 @@ type Lock struct {
 // New returns a ready-to-use Lock. log may be nil; if set, every state
 // transition is logged at debug level.
 func New(log *slog.Logger) *Lock {
-	return &Lock{change: make(chan struct{}), log: log}
+	return &Lock{change: make(chan struct{}), log: log, since: time.Now()}
 }
 
 // broadcast wakes all waiters. Call with mu held.
@@ -63,6 +67,7 @@ func (l *Lock) logTransition(msg string, args ...any) {
 func (l *Lock) SetExternal(holder string) {
 	l.mu.Lock()
 	l.external = holder
+	l.since = time.Now()
 	l.broadcast()
 	l.mu.Unlock()
 	l.logTransition("lock transition", "state", StateExternal, "holder", holder)
@@ -73,6 +78,7 @@ func (l *Lock) SetExternal(holder string) {
 func (l *Lock) ClearExternal() {
 	l.mu.Lock()
 	l.external = ""
+	l.since = time.Now()
 	l.broadcast()
 	l.mu.Unlock()
 	l.logTransition("lock transition", "state", StateIdle)
@@ -90,17 +96,31 @@ func (l *Lock) External() string {
 // while waiting; no state is changed in that case.
 func (l *Lock) AcquireLLM(ctx context.Context) error {
 	l.mu.Lock()
+	waiting := false
 	for l.imageActive || len(l.imageQ) > 0 || l.external != "" {
+		if !waiting {
+			l.llmWaiting++
+			waiting = true
+		}
 		ch := l.change
 		l.mu.Unlock()
 		select {
 		case <-ctx.Done():
+			l.mu.Lock()
+			l.llmWaiting--
+			l.mu.Unlock()
 			return ctx.Err()
 		case <-ch:
 		}
 		l.mu.Lock()
 	}
+	if waiting {
+		l.llmWaiting--
+	}
 	l.n++
+	if l.n == 1 {
+		l.since = time.Now()
+	}
 	n := l.n
 	l.mu.Unlock()
 	l.logTransition("lock transition", "state", StateLLM, "llm_inflight", n)
@@ -129,6 +149,8 @@ func (l *Lock) ReleaseLLM() {
 	l.n--
 	n := l.n
 	if l.n == 0 {
+		l.since = time.Now()
+		l.detail = ""
 		l.broadcast()
 	}
 	l.mu.Unlock()
@@ -156,6 +178,7 @@ func (l *Lock) AcquireImage(ctx context.Context) error {
 		if l.imageQ[0].id == w.id && l.n == 0 && !l.imageActive && l.external == "" {
 			l.imageQ = l.imageQ[1:]
 			l.imageActive = true
+			l.since = time.Now()
 			l.mu.Unlock()
 			l.logTransition("lock transition", "state", StateImage)
 			return nil
@@ -184,6 +207,8 @@ func (l *Lock) AcquireImage(ctx context.Context) error {
 func (l *Lock) ReleaseImage() {
 	l.mu.Lock()
 	l.imageActive = false
+	l.since = time.Now()
+	l.detail = ""
 	l.broadcast()
 	l.mu.Unlock()
 	l.logTransition("lock transition", "state", StateIdle)
@@ -206,3 +231,51 @@ func (l *Lock) Snapshot() (state State, llmInflight int, imagePending bool) {
 	}
 	return state, l.n, l.imageActive || len(l.imageQ) > 0
 }
+
+// SetDetail records what the current holder is doing (e.g. the request
+// path), for status displays. Best effort: overwritten by each new holder,
+// cleared when the GPU goes idle.
+func (l *Lock) SetDetail(detail string) {
+	l.mu.Lock()
+	l.detail = detail
+	l.mu.Unlock()
+}
+
+// Status is a point-in-time view of the lock for monitoring.
+type Status struct {
+	State       State
+	Detail      string
+	LLMInflight int
+	LLMWaiting  int
+	ImageActive bool
+	ImageQueue  int
+	External    string
+	Since       time.Time
+}
+
+// Status reports the full lock state, including waiters and how long the
+// current state has held.
+func (l *Lock) Status() Status {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	s := Status{
+		Detail:      l.detail,
+		LLMInflight: l.n,
+		LLMWaiting:  l.llmWaiting,
+		ImageActive: l.imageActive,
+		ImageQueue:  len(l.imageQ),
+		External:    l.external,
+		Since:       l.since,
+	}
+	switch {
+	case l.imageActive:
+		s.State = StateImage
+	case l.n > 0:
+		s.State = StateLLM
+	case l.external != "":
+		s.State = StateExternal
+	default:
+		s.State = StateIdle
+	}
+	return s
+}

internal/proxy/proxy.go

@@ -491,6 +491,7 @@ func (s *Server) OllamaHandler() http.Handler {
 			return
 		}
 		defer s.cfg.Lock.ReleaseLLM()
+		s.cfg.Lock.SetDetail("ollama: " + r.Method + " " + r.URL.Path)
 		s.ollamaProxy.ServeHTTP(w, r)
 	}))
 }
@@ -590,6 +591,7 @@ func (s *Server) handlePrompt(w http.ResponseWriter, r *http.Request) {
 	}
 	s.cfg.Metrics.ObserveLockWait("image", time.Since(start).Seconds())
 	log.Info("image lock acquired")
+	s.cfg.Lock.SetDetail("comfy: POST /prompt")
 
 	if s.cfg.ComfySup != nil && !comfyFirst {
 		if err := s.cfg.ComfySup.EnsureRunning(); err != nil {

internal/supervise/supervise.go

@@ -164,6 +164,24 @@ func (p *Process) Running() bool {
 	return p.cmd != nil
 }
 
+// Status describes the child for status displays: "external" when something
+// else serves the port, "running" once ready, "starting" while the child
+// boots, "stopped" otherwise.
+func (p *Process) Status() string {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	switch {
+	case p.external:
+		return "external"
+	case p.cmd == nil:
+		return "stopped"
+	case p.ready:
+		return "running"
+	default:
+		return "starting"
+	}
+}
+
 // Ready reports whether the server has answered a probe since its last
 // (re)start. Health checks use it to tell "starting up" from "outage".
 func (p *Process) Ready() bool {