From e58ff339125cabe38e86241ebb93e35b4d768cec Mon Sep 17 00:00:00 2001 From: mram Date: Tue, 22 Sep 2026 16:56:01 +0200 Subject: [PATCH] Monitor: show GPU temperature and fan speed Extends the existing per-tick nvidia-smi query with temperature.gpu and fan.speed (no extra call); N/A values (cards without fan telemetry) are simply omitted from the GPU line. --- cmd/gpu-turnstile/main.go | 46 ++++++++++++++++--------- cmd/gpu-turnstile/monitor.go | 6 ++++ cmd/gpu-turnstile/monitor_test.go | 4 +-- internal/game/game.go | 57 +++++++++++++++++++++++-------- internal/game/game_test.go | 25 ++++++++++++++ 5 files changed, 104 insertions(+), 34 deletions(-) diff --git a/cmd/gpu-turnstile/main.go b/cmd/gpu-turnstile/main.go index b89003d..1c73dbf 100644 --- a/cmd/gpu-turnstile/main.go +++ b/cmd/gpu-turnstile/main.go @@ -847,14 +847,17 @@ type gpuWatch struct { enabled bool usedMB int total int + tempC int + fanPct int known bool foreign string // last Check result: external holders, "" when none at time.Time } -func (g *gpuWatch) setVRAM(used, total int) { +func (g *gpuWatch) setVRAM(st game.GPUStats) { g.mu.Lock() - g.usedMB, g.total, g.known = used, total, true + g.usedMB, g.total = st.UsedMB, st.TotalMB + g.tempC, g.fanPct, g.known = st.TempC, st.FanPct, true g.mu.Unlock() } @@ -864,14 +867,15 @@ func (g *gpuWatch) setCheck(foreign string) { g.mu.Unlock() } -func (g *gpuWatch) get() (used, total int, known bool, foreign string, ageS int64) { +func (g *gpuWatch) get() (st game.GPUStats, known bool, foreign string, ageS int64) { g.mu.Lock() defer g.mu.Unlock() ageS = -1 if !g.at.IsZero() { ageS = int64(time.Since(g.at).Seconds()) } - return g.usedMB, g.total, g.known, g.foreign, ageS + return game.GPUStats{UsedMB: g.usedMB, TotalMB: g.total, TempC: g.tempC, FanPct: g.fanPct}, + g.known, g.foreign, ageS } // gameLoop polls for foreign GPU holders (a game, another ML job). While one @@ -894,8 +898,8 @@ func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *gam log.Warn("game detection failed", "err", err) } gw.setCheck(summarizeHolders(holders)) - if used, total, verr := game.QueryVRAMMB(ctx); verr == nil { - gw.setVRAM(used, total) + if st, verr := game.QueryGPUStats(ctx); verr == nil { + gw.setVRAM(st) } switch { case len(holders) > 0 && !held: @@ -1148,7 +1152,11 @@ type statusGPU struct { Enabled bool `json:"enabled"` UsedMB int `json:"used_mb"` TotalMB int `json:"total_mb"` - Known bool `json:"known"` + // TempC/FanPct are -1 when unknown (never sampled or nvidia-smi + // reported N/A). + TempC int `json:"temp_c"` + FanPct int `json:"fan_pct"` + Known bool `json:"known"` // Foreign is the last detector finding (external GPU holders), empty // when the last check found none. Foreign string `json:"foreign,omitempty"` @@ -1202,12 +1210,16 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces Version: version, UptimeS: int64(time.Since(started).Seconds()), } - used, total, known, foreign, ageS := gw.get() + st, known, foreign, ageS := gw.get() snap.GPU = statusGPU{ Enabled: gw.enabled, - UsedMB: used, TotalMB: total, Known: known, + UsedMB: st.UsedMB, TotalMB: st.TotalMB, Known: known, + TempC: -1, FanPct: -1, Foreign: foreign, AgeS: ageS, } + if known { + snap.GPU.TempC, snap.GPU.FanPct = st.TempC, st.FanPct + } if cfg.OllamaURL != "" { d := statusDownstream{ Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"), @@ -1232,15 +1244,15 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces } snap.Downstreams = append(snap.Downstreams, d) } - st := lk.Status() + lst := lk.Status() snap.Lock = statusLock{ - State: string(st.State), - Detail: st.Detail, - LLMInflight: st.LLMInflight, - LLMWaiting: st.LLMWaiting, - ImageQueue: st.ImageQueue, - External: st.External, - SinceS: int64(time.Since(st.Since).Seconds()), + State: string(lst.State), + Detail: lst.Detail, + LLMInflight: lst.LLMInflight, + LLMWaiting: lst.LLMWaiting, + ImageQueue: lst.ImageQueue, + External: lst.External, + SinceS: int64(time.Since(lst.Since).Seconds()), } b, err := json.Marshal(snap) if err != nil { diff --git a/cmd/gpu-turnstile/monitor.go b/cmd/gpu-turnstile/monitor.go index 8153409..65873a2 100644 --- a/cmd/gpu-turnstile/monitor.go +++ b/cmd/gpu-turnstile/monitor.go @@ -225,6 +225,12 @@ func renderGPU(g statusGPU) string { s := " GPU: " if g.Known { s += renderVRAM(g.UsedMB, g.TotalMB) + if g.TempC >= 0 { + s += fmt.Sprintf(" · %d°C", g.TempC) + } + if g.FanPct >= 0 { + s += fmt.Sprintf(" · fan %d%%", g.FanPct) + } } else { s += cDim + "VRAM unknown (nvidia-smi not answering)" + cReset } diff --git a/cmd/gpu-turnstile/monitor_test.go b/cmd/gpu-turnstile/monitor_test.go index ec5d230..9e1e071 100644 --- a/cmd/gpu-turnstile/monitor_test.go +++ b/cmd/gpu-turnstile/monitor_test.go @@ -32,9 +32,9 @@ func TestRenderMonitor(t *testing.T) { } } - snap.GPU = statusGPU{Enabled: true, Known: true, UsedMB: 4300, TotalMB: 16384, Foreign: "cyberpunk2077.exe (pid 1234)", AgeS: 12} + snap.GPU = statusGPU{Enabled: true, Known: true, UsedMB: 4300, TotalMB: 16384, TempC: 55, FanPct: 42, Foreign: "cyberpunk2077.exe (pid 1234)", AgeS: 12} frame = renderMonitor(snap, 80) - for _, want := range []string{"GPU:", "4.2 GiB / 16.0 GiB used", "external:", "checked 12s ago"} { + for _, want := range []string{"GPU:", "4.2 GiB / 16.0 GiB used", "55°C", "fan 42%", "external:", "checked 12s ago"} { if !strings.Contains(frame, want) { t.Errorf("frame missing %q:\n%s", want, frame) } diff --git a/internal/game/game.go b/internal/game/game.go index da038c9..1b69d6f 100644 --- a/internal/game/game.go +++ b/internal/game/game.go @@ -186,27 +186,54 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) { return parseComputeApps(string(out)) } -// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi. -// Unlike the per-process list this works under WDDM too. -func QueryVRAMMB(ctx context.Context) (used, total int, err error) { +// GPUStats is one nvidia-smi reading of the whole card. +type GPUStats struct { + UsedMB int + TotalMB int + TempC int // -1 when nvidia-smi reports N/A + FanPct int // -1 when N/A (some cards don't expose the fan) +} + +// QueryGPUStats returns VRAM usage, temperature and fan speed via +// nvidia-smi. Unlike the per-process list this works under WDDM too. +func QueryGPUStats(ctx context.Context) (GPUStats, error) { out, err := exec.CommandContext(ctx, "nvidia-smi", - "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output() + "--query-gpu=memory.used,memory.total,temperature.gpu,fan.speed", "--format=csv,noheader,nounits").Output() if err != nil { - return 0, 0, err + return GPUStats{}, err } - usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",") - if !ok { - return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out))) + return parseGPUStats(string(out)) +} + +// parseGPUStats parses one "used, total, temp, fan" CSV line (MiB, °C, +// percent). The memory fields must be numeric; temperature and fan fall +// back to -1 on "N/A" and friends. +func parseGPUStats(out string) (GPUStats, error) { + fields := strings.Split(strings.TrimSpace(out), ",") + if len(fields) != 4 { + return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(out)) } - used, err = strconv.Atoi(strings.TrimSpace(usedStr)) - if err != nil { - return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out))) + num := func(s string) (int, error) { + return strconv.Atoi(strings.TrimSpace(s)) } - total, err = strconv.Atoi(strings.TrimSpace(totalStr)) - if err != nil { - return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out))) + optional := func(s string) int { + n, err := num(s) + if err != nil { + return -1 + } + return n } - return used, total, nil + var st GPUStats + var err error + if st.UsedMB, err = num(fields[0]); err != nil { + return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(out)) + } + if st.TotalMB, err = num(fields[1]); err != nil { + return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(out)) + } + st.TempC = optional(fields[2]) + st.FanPct = optional(fields[3]) + return st, nil } // parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB diff --git a/internal/game/game_test.go b/internal/game/game_test.go index c05c02b..59ea461 100644 --- a/internal/game/game_test.go +++ b/internal/game/game_test.go @@ -116,6 +116,31 @@ func TestParseGPUEngineInstance(t *testing.T) { } } +func TestParseGPUStats(t *testing.T) { + st, err := parseGPUStats("4300, 16384, 55, 42\n") + if err != nil { + t.Fatal(err) + } + if st.UsedMB != 4300 || st.TotalMB != 16384 || st.TempC != 55 || st.FanPct != 42 { + t.Errorf("got %+v", st) + } + + // Cards that don't expose temperature/fan report N/A. + st, err = parseGPUStats("1024, 16384, N/A, N/A") + if err != nil { + t.Fatal(err) + } + if st.TempC != -1 || st.FanPct != -1 { + t.Errorf("got %+v, want -1 for N/A fields", st) + } + + for _, bad := range []string{"", "1, 2", "x, 16384, 55, 42", "1024, x, 55, 42", "1, 2, 3, 4, 5"} { + if _, err := parseGPUStats(bad); err == nil { + t.Errorf("%q parsed, want failure", bad) + } + } +} + func TestProcessesLive(t *testing.T) { if runtime.GOOS != "windows" && runtime.GOOS != "linux" { t.Skip("no process listing on this platform")