PUBLIC/gpu-turnstile
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 19 Wiki Activity

Monitor: show GPU temperature and fan speed

Browse Source 
Extends the existing per-tick nvidia-smi query with temperature.gpu and
fan.speed (no extra call); N/A values (cards without fan telemetry) are
simply omitted from the GPU line.
This commit is contained in:
mram
2026-09-22 16:56:01 +02:00
parent 2040b30c94
commit e58ff33912
5 changed files with 104 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/gpu-turnstile/main.go
+29 -17
View File
@@ -847,14 +847,17 @@ type gpuWatch struct {
enabled bool
usedMB int
total int
tempC int
fanPct int
known bool
foreign string // last Check result: external holders, "" when none
at time.Time
}
func (g *gpuWatch) setVRAM(used, total int) {
func (g *gpuWatch) setVRAM(st game.GPUStats) {
g.mu.Lock()
g.usedMB, g.total, g.known = used, total, true
g.usedMB, g.total = st.UsedMB, st.TotalMB
g.tempC, g.fanPct, g.known = st.TempC, st.FanPct, true
g.mu.Unlock()
}
@@ -864,14 +867,15 @@ func (g *gpuWatch) setCheck(foreign string) {
g.mu.Unlock()
}
func (g *gpuWatch) get() (used, total int, known bool, foreign string, ageS int64) {
func (g *gpuWatch) get() (st game.GPUStats, known bool, foreign string, ageS int64) {
g.mu.Lock()
defer g.mu.Unlock()
ageS = -1
if !g.at.IsZero() {
ageS = int64(time.Since(g.at).Seconds())
}
return g.usedMB, g.total, g.known, g.foreign, ageS
return game.GPUStats{UsedMB: g.usedMB, TotalMB: g.total, TempC: g.tempC, FanPct: g.fanPct},
g.known, g.foreign, ageS
}
// gameLoop polls for foreign GPU holders (a game, another ML job). While one
@@ -894,8 +898,8 @@ func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *gam
log.Warn("game detection failed", "err", err)
}
gw.setCheck(summarizeHolders(holders))
if used, total, verr := game.QueryVRAMMB(ctx); verr == nil {
gw.setVRAM(used, total)
if st, verr := game.QueryGPUStats(ctx); verr == nil {
gw.setVRAM(st)
}
switch {
case len(holders) > 0 && !held:
@@ -1148,7 +1152,11 @@ type statusGPU struct {
Enabled bool `json:"enabled"`
UsedMB int `json:"used_mb"`
TotalMB int `json:"total_mb"`
Known bool `json:"known"`
// TempC/FanPct are -1 when unknown (never sampled or nvidia-smi
// reported N/A).
TempC int `json:"temp_c"`
FanPct int `json:"fan_pct"`
Known bool `json:"known"`
// Foreign is the last detector finding (external GPU holders), empty
// when the last check found none.
Foreign string `json:"foreign,omitempty"`
@@ -1202,12 +1210,16 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces
Version: version,
UptimeS: int64(time.Since(started).Seconds()),
}
used, total, known, foreign, ageS := gw.get()
st, known, foreign, ageS := gw.get()
snap.GPU = statusGPU{
Enabled: gw.enabled,
UsedMB: used, TotalMB: total, Known: known,
UsedMB: st.UsedMB, TotalMB: st.TotalMB, Known: known,
TempC: -1, FanPct: -1,
Foreign: foreign, AgeS: ageS,
}
if known {
snap.GPU.TempC, snap.GPU.FanPct = st.TempC, st.FanPct
}
if cfg.OllamaURL != "" {
d := statusDownstream{
Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"),
@@ -1232,15 +1244,15 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces
}
snap.Downstreams = append(snap.Downstreams, d)
}
st := lk.Status()
lst := lk.Status()
snap.Lock = statusLock{
State: string(st.State),
Detail: st.Detail,
LLMInflight: st.LLMInflight,
LLMWaiting: st.LLMWaiting,
ImageQueue: st.ImageQueue,
External: st.External,
SinceS: int64(time.Since(st.Since).Seconds()),
State: string(lst.State),
Detail: lst.Detail,
LLMInflight: lst.LLMInflight,
LLMWaiting: lst.LLMWaiting,
ImageQueue: lst.ImageQueue,
External: lst.External,
SinceS: int64(time.Since(lst.Since).Seconds()),
}
b, err := json.Marshal(snap)
if err != nil {