From d2c49e52fa3d183c44bb632fab99a1ac63e38f6e Mon Sep 17 00:00:00 2001 From: mram Date: Tue, 22 Sep 2026 12:44:04 +0200 Subject: [PATCH] Monitor: show Ollama's loaded models, their VRAM footprint, and busy state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The status snapshot now queries /api/ps (2s timeout so a wedged Ollama cannot stall the channel) and the ollama line renders e.g. 'UP · llama3.1:8b (4.8 GiB VRAM) · busy'; comfy gets the busy marker too while an image job runs. --- cmd/gpu-turnstile/main.go | 34 +++++++++++++++++++---- cmd/gpu-turnstile/monitor.go | 45 +++++++++++++++++++++++-------- cmd/gpu-turnstile/monitor_test.go | 19 +++++++++++++ internal/ollama/client.go | 39 ++++++++++++++++++++------- 4 files changed, 112 insertions(+), 25 deletions(-) diff --git a/cmd/gpu-turnstile/main.go b/cmd/gpu-turnstile/main.go index 687859d..b89003d 100644 --- a/cmd/gpu-turnstile/main.go +++ b/cmd/gpu-turnstile/main.go @@ -813,7 +813,7 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri // AUTO_UPDATE. if isService { serveControl(ctx, log, u, exePath, applyStaged, - statusProvider(cfg, lk, comfySup, health, started, gw), + statusProvider(cfg, lk, comfySup, health, started, gw, ollamaClient), reloadHandler(cfg, configPath, restartWhenIdle)) } @@ -1106,6 +1106,16 @@ type statusDownstream struct { URL string `json:"url"` Up bool `json:"up"` Managed string `json:"managed,omitempty"` + // Models lists Ollama's loaded models with their VRAM footprint. Null + // when unknown (query failed / not applicable); [] means none loaded — + // deliberately no omitempty so the two stay distinguishable. + Models []statusModel `json:"models"` +} + +// statusModel is one loaded Ollama model. +type statusModel struct { + Name string `json:"name"` + VRAMMB int64 `json:"vram_mb"` // 0 = resident in RAM, not VRAM } type statusLock struct { @@ -1183,8 +1193,10 @@ func diffConfig(a, b config.Config) []string { return out } -// statusProvider assembles the one-line JSON snapshot for CmdStatus. -func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time, gw *gpuWatch) func() string { +// statusProvider assembles the one-line JSON snapshot for CmdStatus. The +// loaded-model query to Ollama gets a short timeout so a wedged upstream +// cannot stall the status channel for long. +func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time, gw *gpuWatch, ollamaClient *ollama.Client) func() string { return func() string { snap := statusSnapshot{ Version: version, @@ -1197,9 +1209,21 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces Foreign: foreign, AgeS: ageS, } if cfg.OllamaURL != "" { - snap.Downstreams = append(snap.Downstreams, statusDownstream{ + d := statusDownstream{ Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"), - }) + } + if ollamaClient != nil { + mctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + models, err := ollamaClient.LoadedModelDetails(mctx) + cancel() + if err == nil { + d.Models = make([]statusModel, 0, len(models)) + for _, m := range models { + d.Models = append(d.Models, statusModel{Name: m.Name, VRAMMB: m.SizeVRAM / (1024 * 1024)}) + } + } + } + snap.Downstreams = append(snap.Downstreams, d) } if cfg.ComfyURL != "" { d := statusDownstream{Name: "comfy", URL: cfg.ComfyURL, Up: health.get("comfy")} diff --git a/cmd/gpu-turnstile/monitor.go b/cmd/gpu-turnstile/monitor.go index d8e0066..365f00b 100644 --- a/cmd/gpu-turnstile/monitor.go +++ b/cmd/gpu-turnstile/monitor.go @@ -188,7 +188,9 @@ func renderMonitor(snap statusSnapshot, width int) string { b.WriteString(cDim + " " + strings.Repeat("─", width-2) + cReset + "\x1b[K\n") for _, d := range snap.Downstreams { - b.WriteString(renderDownstream(d) + "\x1b[K\n") + busy := (d.Name == "ollama" && snap.Lock.State == "llm" && snap.Lock.LLMInflight > 0) || + (d.Name == "comfy" && snap.Lock.State == "image") + b.WriteString(renderDownstream(d, busy) + "\x1b[K\n") } b.WriteString("\x1b[K\n") b.WriteString(renderLock(snap.Lock) + "\x1b[K\n") @@ -227,25 +229,46 @@ func renderGPU(g statusGPU) string { return s } +// formatMB renders MiB as "4.2 GiB" (or MiB below 1 GiB). +func formatMB(mb int64) string { + if mb >= 1024 { + return fmt.Sprintf("%.1f GiB", float64(mb)/1024) + } + return fmt.Sprintf("%d MiB", mb) +} + // renderVRAM renders "4.2 / 16.0 GiB used" (or MiB below 1 GiB). func renderVRAM(used, total int) string { - format := func(mb int) string { - if mb >= 1024 { - return fmt.Sprintf("%.1f GiB", float64(mb)/1024) - } - return fmt.Sprintf("%d MiB", mb) - } if total > 0 { - return format(used) + " / " + format(total) + " used" + return formatMB(int64(used)) + " / " + formatMB(int64(total)) + " used" } - return format(used) + " used" + return formatMB(int64(used)) + " used" } // printableLen counts characters without ANSI escapes (ASCII-only content). func printableLen(s string) int { return len(s) } -func renderDownstream(d statusDownstream) string { +func renderDownstream(d statusDownstream, busy bool) string { url := cDim + d.URL + cReset + detail := "" + if d.Models != nil { + if len(d.Models) == 0 { + detail = cDim + " · no models loaded" + cReset + } else { + parts := make([]string, 0, len(d.Models)) + for _, m := range d.Models { + if m.VRAMMB > 0 { + parts = append(parts, fmt.Sprintf("%s (%s VRAM)", m.Name, formatMB(m.VRAMMB))) + } else { + parts = append(parts, m.Name+" (in RAM)") + } + } + detail = " · " + strings.Join(parts, ", ") + } + } + if busy { + detail += " · " + cCyan + "busy" + cReset + } switch d.Managed { case "stopped": return fmt.Sprintf(" %s○%s %-8s %sstopped (managed — starts on demand)%s %s", @@ -259,7 +282,7 @@ func renderDownstream(d statusDownstream) string { suffix = " (external)" } if d.Up { - return fmt.Sprintf(" %s●%s %-8s %sUP%s%s %s", cGreen, cReset, d.Name, cGreen, cReset, suffix, url) + return fmt.Sprintf(" %s●%s %-8s %sUP%s%s%s %s", cGreen, cReset, d.Name, cGreen, cReset, suffix, detail, url) } return fmt.Sprintf(" %s●%s %-8s %sDOWN%s %s", cRed, cReset, d.Name, cRed, cReset, url) } diff --git a/cmd/gpu-turnstile/monitor_test.go b/cmd/gpu-turnstile/monitor_test.go index b518666..ec5d230 100644 --- a/cmd/gpu-turnstile/monitor_test.go +++ b/cmd/gpu-turnstile/monitor_test.go @@ -47,6 +47,25 @@ func TestRenderMonitor(t *testing.T) { t.Errorf("frame missing %q:\n%s", want, frame) } } + + snap.Downstreams[0].Models = []statusModel{{Name: "llama3.1:8b", VRAMMB: 4900}, {Name: "embed", VRAMMB: 0}} + snap.Lock = statusLock{State: "llm", LLMInflight: 1} + frame = renderMonitor(snap, 80) + for _, want := range []string{"llama3.1:8b (4.8 GiB VRAM)", "embed (in RAM)", "busy"} { + if !strings.Contains(frame, want) { + t.Errorf("frame missing %q:\n%s", want, frame) + } + } + + snap.Downstreams[0].Models = []statusModel{} + snap.Lock = statusLock{State: "idle"} + frame = renderMonitor(snap, 80) + if !strings.Contains(frame, "no models loaded") { + t.Errorf("frame missing %q:\n%s", "no models loaded", frame) + } + if strings.Contains(frame, "busy") { + t.Errorf("idle lock still shows busy:\n%s", frame) + } } func TestFmtDur(t *testing.T) { diff --git a/internal/ollama/client.go b/internal/ollama/client.go index 8c57e72..c7bb5b0 100644 --- a/internal/ollama/client.go +++ b/internal/ollama/client.go @@ -60,13 +60,21 @@ func (c *Client) Probe(ctx context.Context) error { type psResponse struct { Models []struct { - Name string `json:"name"` - Model string `json:"model"` + Name string `json:"name"` + Model string `json:"model"` + SizeVRAM int64 `json:"size_vram"` // bytes resident in VRAM (0 = RAM-only) } `json:"models"` } -// LoadedModels returns the names of models currently held in memory. -func (c *Client) LoadedModels(ctx context.Context) ([]string, error) { +// LoadedModel is one model currently held in memory. +type LoadedModel struct { + Name string + SizeVRAM int64 // bytes resident in VRAM; 0 when the model sits in RAM +} + +// LoadedModelDetails returns the models currently held in memory with +// their VRAM footprint. +func (c *Client) LoadedModelDetails(ctx context.Context) ([]LoadedModel, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/api/ps", nil) if err != nil { return nil, err @@ -84,13 +92,26 @@ func (c *Client) LoadedModels(ctx context.Context) ([]string, error) { if err := json.NewDecoder(resp.Body).Decode(&ps); err != nil { return nil, err } - models := make([]string, 0, len(ps.Models)) + models := make([]LoadedModel, 0, len(ps.Models)) for _, m := range ps.Models { - if m.Name != "" { - models = append(models, m.Name) - } else { - models = append(models, m.Model) + name := m.Name + if name == "" { + name = m.Model } + models = append(models, LoadedModel{Name: name, SizeVRAM: m.SizeVRAM}) + } + return models, nil +} + +// LoadedModels returns the names of models currently held in memory. +func (c *Client) LoadedModels(ctx context.Context) ([]string, error) { + details, err := c.LoadedModelDetails(ctx) + if err != nil { + return nil, err + } + models := make([]string, 0, len(details)) + for _, m := range details { + models = append(models, m.Name) } return models, nil }