Monitor: show GPU temperature and fan speed
Extends the existing per-tick nvidia-smi query with temperature.gpu and fan.speed (no extra call); N/A values (cards without fan telemetry) are simply omitted from the GPU line.
This commit is contained in:
+42
-15
@@ -186,27 +186,54 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
|
||||
return parseComputeApps(string(out))
|
||||
}
|
||||
|
||||
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
|
||||
// Unlike the per-process list this works under WDDM too.
|
||||
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
|
||||
// GPUStats is one nvidia-smi reading of the whole card.
|
||||
type GPUStats struct {
|
||||
UsedMB int
|
||||
TotalMB int
|
||||
TempC int // -1 when nvidia-smi reports N/A
|
||||
FanPct int // -1 when N/A (some cards don't expose the fan)
|
||||
}
|
||||
|
||||
// QueryGPUStats returns VRAM usage, temperature and fan speed via
|
||||
// nvidia-smi. Unlike the per-process list this works under WDDM too.
|
||||
func QueryGPUStats(ctx context.Context) (GPUStats, error) {
|
||||
out, err := exec.CommandContext(ctx, "nvidia-smi",
|
||||
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
|
||||
"--query-gpu=memory.used,memory.total,temperature.gpu,fan.speed", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
return GPUStats{}, err
|
||||
}
|
||||
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
|
||||
return parseGPUStats(string(out))
|
||||
}
|
||||
|
||||
// parseGPUStats parses one "used, total, temp, fan" CSV line (MiB, °C,
|
||||
// percent). The memory fields must be numeric; temperature and fan fall
|
||||
// back to -1 on "N/A" and friends.
|
||||
func parseGPUStats(out string) (GPUStats, error) {
|
||||
fields := strings.Split(strings.TrimSpace(out), ",")
|
||||
if len(fields) != 4 {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(out))
|
||||
}
|
||||
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
|
||||
num := func(s string) (int, error) {
|
||||
return strconv.Atoi(strings.TrimSpace(s))
|
||||
}
|
||||
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
|
||||
optional := func(s string) int {
|
||||
n, err := num(s)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return n
|
||||
}
|
||||
return used, total, nil
|
||||
var st GPUStats
|
||||
var err error
|
||||
if st.UsedMB, err = num(fields[0]); err != nil {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(out))
|
||||
}
|
||||
if st.TotalMB, err = num(fields[1]); err != nil {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(out))
|
||||
}
|
||||
st.TempC = optional(fields[2])
|
||||
st.FanPct = optional(fields[3])
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
|
||||
|
||||
Reference in New Issue
Block a user