Monitor: show GPU temperature and fan speed
Extends the existing per-tick nvidia-smi query with temperature.gpu and fan.speed (no extra call); N/A values (cards without fan telemetry) are simply omitted from the GPU line.
This commit is contained in:
+42
-15
@@ -186,27 +186,54 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
|
||||
return parseComputeApps(string(out))
|
||||
}
|
||||
|
||||
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
|
||||
// Unlike the per-process list this works under WDDM too.
|
||||
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
|
||||
// GPUStats is one nvidia-smi reading of the whole card.
|
||||
type GPUStats struct {
|
||||
UsedMB int
|
||||
TotalMB int
|
||||
TempC int // -1 when nvidia-smi reports N/A
|
||||
FanPct int // -1 when N/A (some cards don't expose the fan)
|
||||
}
|
||||
|
||||
// QueryGPUStats returns VRAM usage, temperature and fan speed via
|
||||
// nvidia-smi. Unlike the per-process list this works under WDDM too.
|
||||
func QueryGPUStats(ctx context.Context) (GPUStats, error) {
|
||||
out, err := exec.CommandContext(ctx, "nvidia-smi",
|
||||
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
|
||||
"--query-gpu=memory.used,memory.total,temperature.gpu,fan.speed", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
return GPUStats{}, err
|
||||
}
|
||||
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
|
||||
return parseGPUStats(string(out))
|
||||
}
|
||||
|
||||
// parseGPUStats parses one "used, total, temp, fan" CSV line (MiB, °C,
|
||||
// percent). The memory fields must be numeric; temperature and fan fall
|
||||
// back to -1 on "N/A" and friends.
|
||||
func parseGPUStats(out string) (GPUStats, error) {
|
||||
fields := strings.Split(strings.TrimSpace(out), ",")
|
||||
if len(fields) != 4 {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(out))
|
||||
}
|
||||
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
|
||||
num := func(s string) (int, error) {
|
||||
return strconv.Atoi(strings.TrimSpace(s))
|
||||
}
|
||||
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
|
||||
optional := func(s string) int {
|
||||
n, err := num(s)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return n
|
||||
}
|
||||
return used, total, nil
|
||||
var st GPUStats
|
||||
var err error
|
||||
if st.UsedMB, err = num(fields[0]); err != nil {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(out))
|
||||
}
|
||||
if st.TotalMB, err = num(fields[1]); err != nil {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(out))
|
||||
}
|
||||
st.TempC = optional(fields[2])
|
||||
st.FanPct = optional(fields[3])
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
|
||||
|
||||
@@ -116,6 +116,31 @@ func TestParseGPUEngineInstance(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGPUStats(t *testing.T) {
|
||||
st, err := parseGPUStats("4300, 16384, 55, 42\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.UsedMB != 4300 || st.TotalMB != 16384 || st.TempC != 55 || st.FanPct != 42 {
|
||||
t.Errorf("got %+v", st)
|
||||
}
|
||||
|
||||
// Cards that don't expose temperature/fan report N/A.
|
||||
st, err = parseGPUStats("1024, 16384, N/A, N/A")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.TempC != -1 || st.FanPct != -1 {
|
||||
t.Errorf("got %+v, want -1 for N/A fields", st)
|
||||
}
|
||||
|
||||
for _, bad := range []string{"", "1, 2", "x, 16384, 55, 42", "1024, x, 55, 42", "1, 2, 3, 4, 5"} {
|
||||
if _, err := parseGPUStats(bad); err == nil {
|
||||
t.Errorf("%q parsed, want failure", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesLive(t *testing.T) {
|
||||
if runtime.GOOS != "windows" && runtime.GOOS != "linux" {
|
||||
t.Skip("no process listing on this platform")
|
||||
|
||||
Reference in New Issue
Block a user