PUBLIC/gpu-turnstile
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 19 Wiki Activity

Monitor: show GPU temperature and fan speed

Browse Source 
Extends the existing per-tick nvidia-smi query with temperature.gpu and
fan.speed (no extra call); N/A values (cards without fan telemetry) are
simply omitted from the GPU line.
This commit is contained in:
mram
2026-09-22 16:56:01 +02:00
parent 2040b30c94
commit e58ff33912
5 changed files with 104 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/gpu-turnstile/main.go
+28 -16
View File
@@ -847,14 +847,17 @@ type gpuWatch struct {
enabled bool
usedMB int
total int
tempC int
fanPct int
known bool
foreign string // last Check result: external holders, "" when none
at time.Time
}
func (g *gpuWatch) setVRAM(used, total int) {
func (g *gpuWatch) setVRAM(st game.GPUStats) {
g.mu.Lock()
g.usedMB, g.total, g.known = used, total, true
g.usedMB, g.total = st.UsedMB, st.TotalMB
g.tempC, g.fanPct, g.known = st.TempC, st.FanPct, true
g.mu.Unlock()
}
@@ -864,14 +867,15 @@ func (g *gpuWatch) setCheck(foreign string) {
g.mu.Unlock()
}
func (g *gpuWatch) get() (used, total int, known bool, foreign string, ageS int64) {
func (g *gpuWatch) get() (st game.GPUStats, known bool, foreign string, ageS int64) {
g.mu.Lock()
defer g.mu.Unlock()
ageS = -1
if !g.at.IsZero() {
ageS = int64(time.Since(g.at).Seconds())
}
return g.usedMB, g.total, g.known, g.foreign, ageS
return game.GPUStats{UsedMB: g.usedMB, TotalMB: g.total, TempC: g.tempC, FanPct: g.fanPct},
g.known, g.foreign, ageS
}
// gameLoop polls for foreign GPU holders (a game, another ML job). While one
@@ -894,8 +898,8 @@ func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *gam
log.Warn("game detection failed", "err", err)
}
gw.setCheck(summarizeHolders(holders))
if used, total, verr := game.QueryVRAMMB(ctx); verr == nil {
gw.setVRAM(used, total)
if st, verr := game.QueryGPUStats(ctx); verr == nil {
gw.setVRAM(st)
}
switch {
case len(holders) > 0 && !held:
@@ -1148,6 +1152,10 @@ type statusGPU struct {
Enabled bool `json:"enabled"`
UsedMB int `json:"used_mb"`
TotalMB int `json:"total_mb"`
// TempC/FanPct are -1 when unknown (never sampled or nvidia-smi
// reported N/A).
TempC int `json:"temp_c"`
FanPct int `json:"fan_pct"`
Known bool `json:"known"`
// Foreign is the last detector finding (external GPU holders), empty
// when the last check found none.
@@ -1202,12 +1210,16 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces
Version: version,
UptimeS: int64(time.Since(started).Seconds()),
}
used, total, known, foreign, ageS := gw.get()
st, known, foreign, ageS := gw.get()
snap.GPU = statusGPU{
Enabled: gw.enabled,
UsedMB: used, TotalMB: total, Known: known,
UsedMB: st.UsedMB, TotalMB: st.TotalMB, Known: known,
TempC: -1, FanPct: -1,
Foreign: foreign, AgeS: ageS,
}
if known {
snap.GPU.TempC, snap.GPU.FanPct = st.TempC, st.FanPct
}
if cfg.OllamaURL != "" {
d := statusDownstream{
Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"),
@@ -1232,15 +1244,15 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces
}
snap.Downstreams = append(snap.Downstreams, d)
}
st := lk.Status()
lst := lk.Status()
snap.Lock = statusLock{
State: string(st.State),
Detail: st.Detail,
LLMInflight: st.LLMInflight,
LLMWaiting: st.LLMWaiting,
ImageQueue: st.ImageQueue,
External: st.External,
SinceS: int64(time.Since(st.Since).Seconds()),
State: string(lst.State),
Detail: lst.Detail,
LLMInflight: lst.LLMInflight,
LLMWaiting: lst.LLMWaiting,
ImageQueue: lst.ImageQueue,
External: lst.External,
SinceS: int64(time.Since(lst.Since).Seconds()),
}
b, err := json.Marshal(snap)
if err != nil {
cmd/gpu-turnstile/monitor.go
+6
View File
@@ -225,6 +225,12 @@ func renderGPU(g statusGPU) string {
s := " GPU: "
if g.Known {
s += renderVRAM(g.UsedMB, g.TotalMB)
if g.TempC >= 0 {
s += fmt.Sprintf(" · %d°C", g.TempC)
}
if g.FanPct >= 0 {
s += fmt.Sprintf(" · fan %d%%", g.FanPct)
}
} else {
s += cDim + "VRAM unknown (nvidia-smi not answering)" + cReset
}
cmd/gpu-turnstile/monitor_test.go
+2 -2
View File
@@ -32,9 +32,9 @@ func TestRenderMonitor(t *testing.T) {
}
}
snap.GPU = statusGPU{Enabled: true, Known: true, UsedMB: 4300, TotalMB: 16384, Foreign: "cyberpunk2077.exe (pid 1234)", AgeS: 12}
snap.GPU = statusGPU{Enabled: true, Known: true, UsedMB: 4300, TotalMB: 16384, TempC: 55, FanPct: 42, Foreign: "cyberpunk2077.exe (pid 1234)", AgeS: 12}
frame = renderMonitor(snap, 80)
for _, want := range []string{"GPU:", "4.2 GiB / 16.0 GiB used", "external:", "checked 12s ago"} {
for _, want := range []string{"GPU:", "4.2 GiB / 16.0 GiB used", "55°C", "fan 42%", "external:", "checked 12s ago"} {
if !strings.Contains(frame, want) {
t.Errorf("frame missing %q:\n%s", want, frame)
}
internal/game/game.go
+41 -14
View File
@@ -186,27 +186,54 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
return parseComputeApps(string(out))
}
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
// Unlike the per-process list this works under WDDM too.
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
// GPUStats is one nvidia-smi reading of the whole card.
type GPUStats struct {
UsedMB int
TotalMB int
TempC int // -1 when nvidia-smi reports N/A
FanPct int // -1 when N/A (some cards don't expose the fan)
}
// QueryGPUStats returns VRAM usage, temperature and fan speed via
// nvidia-smi. Unlike the per-process list this works under WDDM too.
func QueryGPUStats(ctx context.Context) (GPUStats, error) {
out, err := exec.CommandContext(ctx, "nvidia-smi",
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
"--query-gpu=memory.used,memory.total,temperature.gpu,fan.speed", "--format=csv,noheader,nounits").Output()
if err != nil {
return 0, 0, err
return GPUStats{}, err
}
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
if !ok {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
return parseGPUStats(string(out))
}
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
// parseGPUStats parses one "used, total, temp, fan" CSV line (MiB, °C,
// percent). The memory fields must be numeric; temperature and fan fall
// back to -1 on "N/A" and friends.
func parseGPUStats(out string) (GPUStats, error) {
fields := strings.Split(strings.TrimSpace(out), ",")
if len(fields) != 4 {
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(out))
}
num := func(s string) (int, error) {
return strconv.Atoi(strings.TrimSpace(s))
}
optional := func(s string) int {
n, err := num(s)
if err != nil {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
return -1
}
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
if err != nil {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
return n
}
return used, total, nil
var st GPUStats
var err error
if st.UsedMB, err = num(fields[0]); err != nil {
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(out))
}
if st.TotalMB, err = num(fields[1]); err != nil {
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(out))
}
st.TempC = optional(fields[2])
st.FanPct = optional(fields[3])
return st, nil
}
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
internal/game/game_test.go
+25
View File
@@ -116,6 +116,31 @@ func TestParseGPUEngineInstance(t *testing.T) {
}
}
func TestParseGPUStats(t *testing.T) {
st, err := parseGPUStats("4300, 16384, 55, 42\n")
if err != nil {
t.Fatal(err)
}
if st.UsedMB != 4300 || st.TotalMB != 16384 || st.TempC != 55 || st.FanPct != 42 {
t.Errorf("got %+v", st)
}
// Cards that don't expose temperature/fan report N/A.
st, err = parseGPUStats("1024, 16384, N/A, N/A")
if err != nil {
t.Fatal(err)
}
if st.TempC != -1 || st.FanPct != -1 {
t.Errorf("got %+v, want -1 for N/A fields", st)
}
for _, bad := range []string{"", "1, 2", "x, 16384, 55, 42", "1024, x, 55, 42", "1, 2, 3, 4, 5"} {
if _, err := parseGPUStats(bad); err == nil {
t.Errorf("%q parsed, want failure", bad)
}
}
}
func TestProcessesLive(t *testing.T) {
if runtime.GOOS != "windows" && runtime.GOOS != "linux" {
t.Skip("no process listing on this platform")