Monitor: show GPU temperature and fan speed

Extends the existing per-tick nvidia-smi query with temperature.gpu and
fan.speed (no extra call); N/A values (cards without fan telemetry) are
simply omitted from the GPU line.
This commit is contained in:
mram
2026-09-22 16:56:01 +02:00
parent 2040b30c94
commit e58ff33912
5 changed files with 104 additions and 34 deletions
+42 -15
View File
@@ -186,27 +186,54 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
return parseComputeApps(string(out))
}
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
// Unlike the per-process list this works under WDDM too.
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
// GPUStats is one nvidia-smi reading of the whole card.
type GPUStats struct {
UsedMB int
TotalMB int
TempC int // -1 when nvidia-smi reports N/A
FanPct int // -1 when N/A (some cards don't expose the fan)
}
// QueryGPUStats returns VRAM usage, temperature and fan speed via
// nvidia-smi. Unlike the per-process list this works under WDDM too.
func QueryGPUStats(ctx context.Context) (GPUStats, error) {
out, err := exec.CommandContext(ctx, "nvidia-smi",
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
"--query-gpu=memory.used,memory.total,temperature.gpu,fan.speed", "--format=csv,noheader,nounits").Output()
if err != nil {
return 0, 0, err
return GPUStats{}, err
}
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
if !ok {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
return parseGPUStats(string(out))
}
// parseGPUStats parses one "used, total, temp, fan" CSV line (MiB, °C,
// percent). The memory fields must be numeric; temperature and fan fall
// back to -1 on "N/A" and friends.
func parseGPUStats(out string) (GPUStats, error) {
fields := strings.Split(strings.TrimSpace(out), ",")
if len(fields) != 4 {
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(out))
}
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
if err != nil {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
num := func(s string) (int, error) {
return strconv.Atoi(strings.TrimSpace(s))
}
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
if err != nil {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
optional := func(s string) int {
n, err := num(s)
if err != nil {
return -1
}
return n
}
return used, total, nil
var st GPUStats
var err error
if st.UsedMB, err = num(fields[0]); err != nil {
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(out))
}
if st.TotalMB, err = num(fields[1]); err != nil {
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(out))
}
st.TempC = optional(fields[2])
st.FanPct = optional(fields[3])
return st, nil
}
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
+25
View File
@@ -116,6 +116,31 @@ func TestParseGPUEngineInstance(t *testing.T) {
}
}
func TestParseGPUStats(t *testing.T) {
st, err := parseGPUStats("4300, 16384, 55, 42\n")
if err != nil {
t.Fatal(err)
}
if st.UsedMB != 4300 || st.TotalMB != 16384 || st.TempC != 55 || st.FanPct != 42 {
t.Errorf("got %+v", st)
}
// Cards that don't expose temperature/fan report N/A.
st, err = parseGPUStats("1024, 16384, N/A, N/A")
if err != nil {
t.Fatal(err)
}
if st.TempC != -1 || st.FanPct != -1 {
t.Errorf("got %+v, want -1 for N/A fields", st)
}
for _, bad := range []string{"", "1, 2", "x, 16384, 55, 42", "1024, x, 55, 42", "1, 2, 3, 4, 5"} {
if _, err := parseGPUStats(bad); err == nil {
t.Errorf("%q parsed, want failure", bad)
}
}
}
func TestProcessesLive(t *testing.T) {
if runtime.GOOS != "windows" && runtime.GOOS != "linux" {
t.Skip("no process listing on this platform")