Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
31ee6013db | ||
|
|
5506163493 | ||
|
|
18a7336c02 | ||
|
|
0796092b80 | ||
|
|
897163042c | ||
|
|
e58ff33912 | ||
|
|
2040b30c94 |
@@ -49,6 +49,8 @@ override file values. Invalid values fail at startup.
|
||||
|---|---|---|
|
||||
| `LISTEN_OLLAMA` | `:11434` | Listener for Ollama-compatible clients |
|
||||
| `LISTEN_COMFY` | `:8188` | Listener for ComfyUI clients |
|
||||
| `LISTEN_UI` | _(empty = disabled)_ | Web UI listener (`host:port`) mirroring `--monitor` (live status, update/reload buttons). Keep it on `127.0.0.1` unless you set auth |
|
||||
| `UI_USER` / `UI_PASS` | _(empty = no auth)_ | HTTP basic auth for the web UI; must be set together |
|
||||
| `OLLAMA_URL` | _(empty = disabled)_ | Ollama upstream; set to enable the Ollama consumer |
|
||||
| `COMFY_URL` | _(empty = disabled)_ | ComfyUI upstream; set to enable the ComfyUI consumer |
|
||||
| `UNLOAD_TIMEOUT` | `60s` | Wait for Ollama to unload before an image job |
|
||||
|
||||
@@ -221,6 +221,8 @@ override file values. A missing file is fine; a malformed one is fatal.
|
||||
|---|---|---|
|
||||
| `LISTEN_OLLAMA` | `:11434` | listener for Ollama-compatible clients |
|
||||
| `LISTEN_COMFY` | `:8188` | listener for ComfyUI clients |
|
||||
| `LISTEN_UI` | _(empty = disabled)_ | web UI listener (`host:port`) mirroring `--monitor` (live status, update/reload buttons); keep it on `127.0.0.1` unless auth is set |
|
||||
| `UI_USER` / `UI_PASS` | _(empty = no auth)_ | HTTP basic auth for the web UI; must be set together |
|
||||
| `OLLAMA_URL` | _(empty = disabled)_ | Ollama upstream; set to enable the Ollama consumer |
|
||||
| `COMFY_URL` | _(empty = disabled)_ | ComfyUI upstream; set to enable the ComfyUI consumer |
|
||||
| `UNLOAD_TIMEOUT` | `60s` | wait for Ollama to unload |
|
||||
|
||||
+70
-32
@@ -567,6 +567,8 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
||||
"version", version,
|
||||
"listen_ollama", cfg.ListenOllama,
|
||||
"listen_comfy", cfg.ListenComfy,
|
||||
"listen_ui", orDisabled(cfg.ListenUI),
|
||||
"ui_auth", cfg.UIUser != "",
|
||||
"ollama_url", orDisabled(cfg.OllamaURL),
|
||||
"comfy_url", orDisabled(cfg.ComfyURL),
|
||||
"unload_timeout", cfg.UnloadTimeout,
|
||||
@@ -808,13 +810,27 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
||||
}
|
||||
}
|
||||
|
||||
// The control channel (status for --monitor, update-now and reload-env
|
||||
// triggers) is served whenever running as a service, independent of
|
||||
// AUTO_UPDATE.
|
||||
// The command handler backs both the local control channel (served
|
||||
// whenever running as a service) and the optional web UI (LISTEN_UI),
|
||||
// independent of AUTO_UPDATE.
|
||||
handler := controlHandler(ctx, u, exePath, applyStaged,
|
||||
statusProvider(cfg, lk, comfySup, health, started, gw, ollamaClient),
|
||||
reloadHandler(cfg, configPath, restartWhenIdle))
|
||||
if isService {
|
||||
serveControl(ctx, log, u, exePath, applyStaged,
|
||||
statusProvider(cfg, lk, comfySup, health, started, gw, ollamaClient),
|
||||
reloadHandler(cfg, configPath, restartWhenIdle))
|
||||
serveControl(ctx, log, handler)
|
||||
}
|
||||
if cfg.ListenUI != "" {
|
||||
ln, err := net.Listen("tcp", cfg.ListenUI)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen ui on %s: %w", cfg.ListenUI, err)
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(cfg.ListenUI); err == nil && !isLoopbackHost(host) && cfg.UIUser == "" {
|
||||
log.Warn("web UI is reachable from other machines WITHOUT auth; set UI_USER/UI_PASS or bind to 127.0.0.1", "addr", cfg.ListenUI)
|
||||
}
|
||||
uiSrv := &http.Server{Addr: cfg.ListenUI, Handler: uiHandler(handler, cfg.UIUser, cfg.UIPass)}
|
||||
servers = append(servers, uiSrv)
|
||||
log.Warn("listening", "consumer", "ui", "addr", cfg.ListenUI)
|
||||
go func() { errCh <- uiSrv.Serve(ln) }()
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -847,14 +863,17 @@ type gpuWatch struct {
|
||||
enabled bool
|
||||
usedMB int
|
||||
total int
|
||||
tempC int
|
||||
fanPct int
|
||||
known bool
|
||||
foreign string // last Check result: external holders, "" when none
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func (g *gpuWatch) setVRAM(used, total int) {
|
||||
func (g *gpuWatch) setVRAM(st game.GPUStats) {
|
||||
g.mu.Lock()
|
||||
g.usedMB, g.total, g.known = used, total, true
|
||||
g.usedMB, g.total = st.UsedMB, st.TotalMB
|
||||
g.tempC, g.fanPct, g.known = st.TempC, st.FanPct, true
|
||||
g.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -864,14 +883,15 @@ func (g *gpuWatch) setCheck(foreign string) {
|
||||
g.mu.Unlock()
|
||||
}
|
||||
|
||||
func (g *gpuWatch) get() (used, total int, known bool, foreign string, ageS int64) {
|
||||
func (g *gpuWatch) get() (st game.GPUStats, known bool, foreign string, ageS int64) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
ageS = -1
|
||||
if !g.at.IsZero() {
|
||||
ageS = int64(time.Since(g.at).Seconds())
|
||||
}
|
||||
return g.usedMB, g.total, g.known, g.foreign, ageS
|
||||
return game.GPUStats{UsedMB: g.usedMB, TotalMB: g.total, TempC: g.tempC, FanPct: g.fanPct},
|
||||
g.known, g.foreign, ageS
|
||||
}
|
||||
|
||||
// gameLoop polls for foreign GPU holders (a game, another ML job). While one
|
||||
@@ -894,8 +914,8 @@ func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *gam
|
||||
log.Warn("game detection failed", "err", err)
|
||||
}
|
||||
gw.setCheck(summarizeHolders(holders))
|
||||
if used, total, verr := game.QueryVRAMMB(ctx); verr == nil {
|
||||
gw.setVRAM(used, total)
|
||||
if st, verr := game.QueryGPUStats(ctx); verr == nil {
|
||||
gw.setVRAM(st)
|
||||
}
|
||||
switch {
|
||||
case len(holders) > 0 && !held:
|
||||
@@ -1020,17 +1040,18 @@ func updateLoop(ctx context.Context, interval time.Duration, log *slog.Logger, u
|
||||
}
|
||||
}
|
||||
|
||||
// serveControl opens the local control channel (named pipe on Windows,
|
||||
// unix socket on Linux) so unprivileged local users can query status
|
||||
// (--monitor), trigger an update check (--force-update/--update-now) and
|
||||
// poke a config reload (--reload-env) without admin rights. The update
|
||||
// payload is signature-verified regardless of who asks; update triggers
|
||||
// are rate-limited to one per minute so the channel cannot be used to spam
|
||||
// restarts. u is nil when AUTO_UPDATE=false.
|
||||
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string), status func() string, reload func() string) {
|
||||
// controlHandler builds the command handler shared by the control channel
|
||||
// (named pipe / unix socket) and the web UI's action endpoints: status for
|
||||
// --monitor / the UI, an update check (--force-update/--update-now) and a
|
||||
// config reload (--reload-env), all safe for unprivileged local users. The
|
||||
// update payload is signature-verified regardless of who asks; update
|
||||
// triggers are rate-limited to one per minute and reloads to one per two
|
||||
// seconds so neither can be used to spam restarts or disk churn. u is nil
|
||||
// when AUTO_UPDATE=false.
|
||||
func controlHandler(ctx context.Context, u *update.Updater, exePath string, applyStaged func(to string), status func() string, reload func() string) control.Handler {
|
||||
var mu sync.Mutex
|
||||
var lastTrigger, lastReload time.Time
|
||||
h := func(cmd string) string {
|
||||
return func(cmd string) string {
|
||||
switch cmd {
|
||||
case control.CmdStatus:
|
||||
return "OK " + status()
|
||||
@@ -1071,6 +1092,12 @@ func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exeP
|
||||
applyStaged(to)
|
||||
return "OK updated from " + version + " to " + to + "; the service restarts once the GPU is idle"
|
||||
}
|
||||
}
|
||||
|
||||
// serveControl opens the local control channel (named pipe on Windows,
|
||||
// unix socket on Linux) for unprivileged local users (--monitor,
|
||||
// --force-update/--update-now, --reload-env).
|
||||
func serveControl(ctx context.Context, log *slog.Logger, h control.Handler) {
|
||||
if err := control.Serve(ctx, h, log); err != nil {
|
||||
log.Warn("control channel disabled", "err", err)
|
||||
}
|
||||
@@ -1137,6 +1164,8 @@ type statusSnapshot struct {
|
||||
// false when game detection (and with it nvidia-smi polling) is not
|
||||
// configured.
|
||||
GPU statusGPU `json:"gpu"`
|
||||
// UI is the web UI's URL when LISTEN_UI is active, empty otherwise.
|
||||
UI string `json:"ui,omitempty"`
|
||||
// MonitorNote is set client-side (never over the wire) when the
|
||||
// monitor's own binary differs from the service's version.
|
||||
MonitorNote string `json:"-"`
|
||||
@@ -1148,7 +1177,11 @@ type statusGPU struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
UsedMB int `json:"used_mb"`
|
||||
TotalMB int `json:"total_mb"`
|
||||
Known bool `json:"known"`
|
||||
// TempC/FanPct are -1 when unknown (never sampled or nvidia-smi
|
||||
// reported N/A).
|
||||
TempC int `json:"temp_c"`
|
||||
FanPct int `json:"fan_pct"`
|
||||
Known bool `json:"known"`
|
||||
// Foreign is the last detector finding (external GPU holders), empty
|
||||
// when the last check found none.
|
||||
Foreign string `json:"foreign,omitempty"`
|
||||
@@ -1201,13 +1234,18 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces
|
||||
snap := statusSnapshot{
|
||||
Version: version,
|
||||
UptimeS: int64(time.Since(started).Seconds()),
|
||||
UI: uiURL(cfg.ListenUI),
|
||||
}
|
||||
used, total, known, foreign, ageS := gw.get()
|
||||
st, known, foreign, ageS := gw.get()
|
||||
snap.GPU = statusGPU{
|
||||
Enabled: gw.enabled,
|
||||
UsedMB: used, TotalMB: total, Known: known,
|
||||
UsedMB: st.UsedMB, TotalMB: st.TotalMB, Known: known,
|
||||
TempC: -1, FanPct: -1,
|
||||
Foreign: foreign, AgeS: ageS,
|
||||
}
|
||||
if known {
|
||||
snap.GPU.TempC, snap.GPU.FanPct = st.TempC, st.FanPct
|
||||
}
|
||||
if cfg.OllamaURL != "" {
|
||||
d := statusDownstream{
|
||||
Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"),
|
||||
@@ -1232,15 +1270,15 @@ func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Proces
|
||||
}
|
||||
snap.Downstreams = append(snap.Downstreams, d)
|
||||
}
|
||||
st := lk.Status()
|
||||
lst := lk.Status()
|
||||
snap.Lock = statusLock{
|
||||
State: string(st.State),
|
||||
Detail: st.Detail,
|
||||
LLMInflight: st.LLMInflight,
|
||||
LLMWaiting: st.LLMWaiting,
|
||||
ImageQueue: st.ImageQueue,
|
||||
External: st.External,
|
||||
SinceS: int64(time.Since(st.Since).Seconds()),
|
||||
State: string(lst.State),
|
||||
Detail: lst.Detail,
|
||||
LLMInflight: lst.LLMInflight,
|
||||
LLMWaiting: lst.LLMWaiting,
|
||||
ImageQueue: lst.ImageQueue,
|
||||
External: lst.External,
|
||||
SinceS: int64(time.Since(lst.Since).Seconds()),
|
||||
}
|
||||
b, err := json.Marshal(snap)
|
||||
if err != nil {
|
||||
|
||||
@@ -22,13 +22,14 @@ const (
|
||||
)
|
||||
|
||||
// hotkeysLine is the monitor's footer.
|
||||
const hotkeysLine = " " + cDim + "q quit · u update now" + cReset + "\x1b[K\n"
|
||||
const hotkeysLine = " " + cDim + "q quit · u update now · r reload config" + cReset + "\x1b[K\n"
|
||||
|
||||
// monitorCommand renders a live status view of the running service,
|
||||
// refreshed every second from the control channel. When the service
|
||||
// reports a different version and the executable on disk changed (the
|
||||
// updater replaced it), the monitor restarts itself onto the new binary.
|
||||
// Hotkeys: q quits, u triggers an update check on the service.
|
||||
// Hotkeys: q quits, u triggers an update check on the service, r asks the
|
||||
// service to reload its config file.
|
||||
func monitorCommand() int {
|
||||
if !stdoutIsTerminal() {
|
||||
fmt.Fprintln(os.Stderr, "gpu-turnstile: --monitor needs an interactive terminal")
|
||||
@@ -57,7 +58,27 @@ func monitorCommand() int {
|
||||
var note string
|
||||
var noteAt time.Time
|
||||
noteCh := make(chan string, 1)
|
||||
updatePending := false
|
||||
askPending := false
|
||||
|
||||
// ask sends a one-shot command to the service and reports the reply in
|
||||
// the note line. Only one ask runs at a time.
|
||||
ask := func(cmd, busy, label string) {
|
||||
if askPending {
|
||||
return
|
||||
}
|
||||
askPending = true
|
||||
note, noteAt = busy, time.Now()
|
||||
go func() {
|
||||
reply, err := control.Ask(cmd)
|
||||
if err != nil {
|
||||
noteCh <- label + ": no answer from the service"
|
||||
return
|
||||
}
|
||||
msg := strings.TrimPrefix(reply, "OK ")
|
||||
msg = strings.TrimPrefix(msg, "ERR ")
|
||||
noteCh <- label + ": " + msg
|
||||
}()
|
||||
}
|
||||
|
||||
poll := func() string {
|
||||
frame := renderWaiting()
|
||||
@@ -104,23 +125,12 @@ func monitorCommand() int {
|
||||
case 'q', 'Q', 3: // q or Ctrl+C (raw mode delivers it as a byte)
|
||||
return 0
|
||||
case 'u', 'U':
|
||||
if !updatePending {
|
||||
updatePending = true
|
||||
note, noteAt = "checking for updates…", time.Now()
|
||||
go func() {
|
||||
reply, err := control.Ask(control.CmdUpdateNow)
|
||||
if err != nil {
|
||||
noteCh <- "update: no answer from the service"
|
||||
return
|
||||
}
|
||||
msg := strings.TrimPrefix(reply, "OK ")
|
||||
msg = strings.TrimPrefix(msg, "ERR ")
|
||||
noteCh <- "update: " + msg
|
||||
}()
|
||||
}
|
||||
ask(control.CmdUpdateNow, "checking for updates…", "update")
|
||||
case 'r', 'R':
|
||||
ask(control.CmdReloadEnv, "reloading config…", "reload")
|
||||
}
|
||||
case n := <-noteCh:
|
||||
updatePending = false
|
||||
askPending = false
|
||||
note, noteAt = n, time.Now()
|
||||
}
|
||||
}
|
||||
@@ -201,6 +211,9 @@ func renderMonitor(snap statusSnapshot, width int) string {
|
||||
if snap.GPU.Enabled {
|
||||
b.WriteString(renderGPU(snap.GPU) + "\x1b[K\n")
|
||||
}
|
||||
if snap.UI != "" {
|
||||
b.WriteString(" " + cDim + "UI: " + snap.UI + cReset + "\x1b[K\n")
|
||||
}
|
||||
if snap.MonitorNote != "" {
|
||||
b.WriteString(" " + cYellow + snap.MonitorNote + cReset + "\x1b[K\n")
|
||||
}
|
||||
@@ -215,6 +228,12 @@ func renderGPU(g statusGPU) string {
|
||||
s := " GPU: "
|
||||
if g.Known {
|
||||
s += renderVRAM(g.UsedMB, g.TotalMB)
|
||||
if g.TempC >= 0 {
|
||||
s += fmt.Sprintf(" · %d°C", g.TempC)
|
||||
}
|
||||
if g.FanPct >= 0 {
|
||||
s += fmt.Sprintf(" · fan %d%%", g.FanPct)
|
||||
}
|
||||
} else {
|
||||
s += cDim + "VRAM unknown (nvidia-smi not answering)" + cReset
|
||||
}
|
||||
|
||||
@@ -32,9 +32,9 @@ func TestRenderMonitor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
snap.GPU = statusGPU{Enabled: true, Known: true, UsedMB: 4300, TotalMB: 16384, Foreign: "cyberpunk2077.exe (pid 1234)", AgeS: 12}
|
||||
snap.GPU = statusGPU{Enabled: true, Known: true, UsedMB: 4300, TotalMB: 16384, TempC: 55, FanPct: 42, Foreign: "cyberpunk2077.exe (pid 1234)", AgeS: 12}
|
||||
frame = renderMonitor(snap, 80)
|
||||
for _, want := range []string{"GPU:", "4.2 GiB / 16.0 GiB used", "external:", "checked 12s ago"} {
|
||||
for _, want := range []string{"GPU:", "4.2 GiB / 16.0 GiB used", "55°C", "fan 42%", "external:", "checked 12s ago"} {
|
||||
if !strings.Contains(frame, want) {
|
||||
t.Errorf("frame missing %q:\n%s", want, frame)
|
||||
}
|
||||
@@ -66,6 +66,12 @@ func TestRenderMonitor(t *testing.T) {
|
||||
if strings.Contains(frame, "busy") {
|
||||
t.Errorf("idle lock still shows busy:\n%s", frame)
|
||||
}
|
||||
|
||||
snap.UI = "http://127.0.0.1:7860"
|
||||
frame = renderMonitor(snap, 80)
|
||||
if !strings.Contains(frame, "UI: http://127.0.0.1:7860") {
|
||||
t.Errorf("frame missing UI line:\n%s", frame)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFmtDur(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gpu-turnstile/internal/control"
|
||||
)
|
||||
|
||||
// uiHandler serves the web UI (LISTEN_UI): a single page mirroring
|
||||
// --monitor, its status JSON (the same snapshot the control channel
|
||||
// serves) and action endpoints for the update/reload buttons. user/pass
|
||||
// (UI_USER/UI_PASS) enable HTTP basic auth; both empty means open access,
|
||||
// which is meant for localhost only.
|
||||
func uiHandler(h control.Handler, user, pass string) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprint(w, uiPage)
|
||||
})
|
||||
mux.HandleFunc("GET /api/status", func(w http.ResponseWriter, r *http.Request) {
|
||||
msg := strings.TrimPrefix(h(control.CmdStatus), "OK ")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if strings.HasPrefix(msg, "ERR ") {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
msg = strings.TrimPrefix(msg, "ERR ")
|
||||
}
|
||||
fmt.Fprint(w, msg)
|
||||
})
|
||||
mux.HandleFunc("POST /api/action/{cmd}", func(w http.ResponseWriter, r *http.Request) {
|
||||
cmd := r.PathValue("cmd")
|
||||
if cmd != control.CmdUpdateNow && cmd != control.CmdReloadEnv {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
msg := strings.TrimPrefix(h(cmd), "OK ")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
if strings.HasPrefix(msg, "ERR ") {
|
||||
w.WriteHeader(http.StatusConflict)
|
||||
msg = strings.TrimPrefix(msg, "ERR ")
|
||||
}
|
||||
fmt.Fprint(w, msg)
|
||||
})
|
||||
if user != "" {
|
||||
return basicAuth(mux, user, pass)
|
||||
}
|
||||
return mux
|
||||
}
|
||||
|
||||
// basicAuth wraps next with HTTP basic auth (constant-time compares).
|
||||
func basicAuth(next http.Handler, user, pass string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
u, p, ok := r.BasicAuth()
|
||||
if !ok ||
|
||||
subtle.ConstantTimeCompare([]byte(u), []byte(user)) != 1 ||
|
||||
subtle.ConstantTimeCompare([]byte(p), []byte(pass)) != 1 {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="gpu-turnstile"`)
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// isLoopbackHost reports whether a listen host is localhost-only. An empty
|
||||
// host (":7860") binds all interfaces and is therefore not loopback.
|
||||
func isLoopbackHost(host string) bool {
|
||||
if host == "localhost" {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
|
||||
// uiURL renders a LISTEN_UI address as a browsable URL ("" when disabled).
|
||||
// A wildcard host reads as 127.0.0.1 — that's where the browser is usually
|
||||
// running.
|
||||
func uiURL(addr string) string {
|
||||
if addr == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(addr, ":") {
|
||||
addr = "127.0.0.1" + addr
|
||||
}
|
||||
return "http://" + addr
|
||||
}
|
||||
|
||||
// uiPage is the whole web UI: a terminal-styled page that polls /api/status
|
||||
// once a second and renders the same lines as --monitor. The buttons (and
|
||||
// the u/r keys) hit the action endpoints, like the monitor's hotkeys.
|
||||
const uiPage = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>gpu-turnstile</title>
|
||||
<style>
|
||||
body { background:#111; color:#ddd; font:14px/1.6 ui-monospace,Consolas,"Cascadia Mono",monospace;
|
||||
margin:0 auto; padding:14px 18px; max-width:860px; }
|
||||
header { display:flex; justify-content:space-between; border-bottom:1px solid #333;
|
||||
padding-bottom:4px; margin-bottom:10px; }
|
||||
footer { border-top:1px solid #333; margin-top:10px; padding-top:8px;
|
||||
display:flex; gap:8px; align-items:center; }
|
||||
button { background:#1d1d1d; color:#bbb; border:1px solid #444; border-radius:4px;
|
||||
padding:2px 10px; font:inherit; cursor:pointer; }
|
||||
button:hover { background:#2c2c2c; color:#eee; }
|
||||
.dim { color:#777; } .up { color:#4caf50; } .down { color:#ef5350; }
|
||||
.llm { color:#26c6da; } .warn { color:#fbc02d; }
|
||||
#note { color:#fbc02d; min-height:1.6em; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header><span>gpu-turnstile <span id="up" class="dim"></span></span><span id="ver" class="dim"></span></header>
|
||||
<div id="downstreams"></div>
|
||||
<div id="lock"></div>
|
||||
<div id="gpu"></div>
|
||||
<div id="note"></div>
|
||||
<footer>
|
||||
<button id="bu">u update now</button>
|
||||
<button id="br">r reload config</button>
|
||||
<span class="dim">refreshes every second</span>
|
||||
</footer>
|
||||
<script>
|
||||
const $ = id => document.getElementById(id);
|
||||
const el = (cls, text) => { const e = document.createElement("span"); if (cls) e.className = cls; e.textContent = text; return e; };
|
||||
const line = (...parts) => { const d = document.createElement("div"); d.append(...parts); return d; };
|
||||
const fmtMB = mb => mb >= 1024 ? (mb / 1024).toFixed(1) + " GiB" : mb + " MiB";
|
||||
const fmtDur = s => {
|
||||
s = Math.max(0, Math.floor(s));
|
||||
if (s >= 3600) return Math.floor(s / 3600) + "h" + String(Math.floor(s / 60) % 60).padStart(2, "0") + "m";
|
||||
if (s >= 60) return Math.floor(s / 60) + "m" + String(s % 60).padStart(2, "0") + "s";
|
||||
return s + "s";
|
||||
};
|
||||
|
||||
function renderDownstream(d, lock) {
|
||||
const busy = (d.name === "ollama" && lock.state === "llm" && lock.llm_inflight > 0) ||
|
||||
(d.name === "comfy" && lock.state === "image");
|
||||
if (d.managed === "stopped")
|
||||
return line(el("dim", "○ " + d.name + " stopped (managed — starts on demand) " + d.url));
|
||||
if (d.managed === "starting")
|
||||
return line(el("warn", "◌ " + d.name + " starting… "), el("dim", d.url));
|
||||
const parts = [el(d.up ? "up" : "down", "● " + d.name + " " + (d.up ? "UP" : "DOWN"))];
|
||||
if (d.managed === "external") parts.push(el("", " (external)"));
|
||||
if (d.models) {
|
||||
if (d.models.length === 0) {
|
||||
parts.push(el("dim", " · no models loaded"));
|
||||
} else {
|
||||
parts.push(el("", " · " + d.models.map(m =>
|
||||
m.vram_mb > 0 ? m.name + " (" + fmtMB(m.vram_mb) + " VRAM)" : m.name + " (in RAM)").join(", ")));
|
||||
}
|
||||
}
|
||||
if (busy) parts.push(el("llm", " · busy"));
|
||||
parts.push(el("dim", " " + d.url));
|
||||
return line(...parts);
|
||||
}
|
||||
|
||||
function renderLock(l) {
|
||||
const dur = el("dim", " (" + fmtDur(l.since_s || 0) + ")");
|
||||
switch (l.state) {
|
||||
case "idle": return line(el("up", "Lock: idle"), dur);
|
||||
case "llm": {
|
||||
let s = "Lock: LLM — " + l.llm_inflight + " in flight";
|
||||
if (l.llm_waiting > 0) s += ", " + l.llm_waiting + " waiting";
|
||||
if (l.detail) s += " — " + l.detail;
|
||||
return line(el("llm", s), dur);
|
||||
}
|
||||
case "image": return line(el("warn", "Lock: IMAGE" + (l.detail ? " — " + l.detail : "")), dur);
|
||||
case "external": return line(el("down", "Lock: EXTERNAL — " + (l.external || "")), dur);
|
||||
}
|
||||
return line(el("", "Lock: unknown"));
|
||||
}
|
||||
|
||||
function renderGPU(g) {
|
||||
const parts = [el("", "GPU: ")];
|
||||
if (g.known) {
|
||||
let s = fmtMB(g.used_mb) + " / " + fmtMB(g.total_mb) + " used";
|
||||
if (g.temp_c >= 0) s += " · " + g.temp_c + "°C";
|
||||
if (g.fan_pct >= 0) s += " · fan " + g.fan_pct + "%";
|
||||
parts.push(el("", s));
|
||||
} else {
|
||||
parts.push(el("dim", "VRAM unknown (nvidia-smi not answering)"));
|
||||
}
|
||||
parts.push(g.foreign ? el("down", " · external: " + g.foreign) : el("dim", " · no external process"));
|
||||
if (g.age_s >= 0) parts.push(el("dim", " (checked " + fmtDur(g.age_s) + " ago)"));
|
||||
return line(...parts);
|
||||
}
|
||||
|
||||
async function poll() {
|
||||
let snap;
|
||||
try {
|
||||
const r = await fetch("/api/status");
|
||||
if (!r.ok) throw new Error();
|
||||
snap = await r.json();
|
||||
} catch {
|
||||
$("downstreams").replaceChildren(line(el("dim", "gpu-turnstile — waiting for a running service…")));
|
||||
$("lock").replaceChildren(); $("gpu").replaceChildren();
|
||||
return;
|
||||
}
|
||||
$("ver").textContent = snap.version || "";
|
||||
$("up").textContent = snap.uptime_s > 0 ? "up " + fmtDur(snap.uptime_s) : "";
|
||||
const ds = $("downstreams"); ds.replaceChildren();
|
||||
for (const d of snap.downstreams || []) ds.appendChild(renderDownstream(d, snap.lock || {}));
|
||||
const lk = $("lock"); lk.replaceChildren(renderLock(snap.lock || {}));
|
||||
if ((snap.lock || {}).image_queue > 0)
|
||||
lk.appendChild(line(el("warn", "Queue: " + snap.lock.image_queue + " image job(s) waiting")));
|
||||
const g = $("gpu"); g.replaceChildren();
|
||||
if (snap.gpu && snap.gpu.enabled) g.appendChild(renderGPU(snap.gpu));
|
||||
}
|
||||
|
||||
let noteTimer;
|
||||
async function act(cmd) {
|
||||
const note = $("note");
|
||||
note.textContent = "…";
|
||||
try {
|
||||
const r = await fetch("/api/action/" + cmd, { method: "POST" });
|
||||
note.textContent = await r.text();
|
||||
} catch {
|
||||
note.textContent = "no answer from the service";
|
||||
}
|
||||
clearTimeout(noteTimer);
|
||||
noteTimer = setTimeout(() => { note.textContent = ""; }, 15000);
|
||||
}
|
||||
|
||||
$("bu").onclick = () => act("update-now");
|
||||
$("br").onclick = () => act("reload-env");
|
||||
document.addEventListener("keydown", e => {
|
||||
if (e.key === "u") act("update-now");
|
||||
if (e.key === "r") act("reload-env");
|
||||
});
|
||||
poll();
|
||||
setInterval(poll, 1000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
@@ -0,0 +1,185 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gpu-turnstile/internal/control"
|
||||
)
|
||||
|
||||
func testUIHandler() *httptest.Server {
|
||||
h := uiHandler(func(cmd string) string {
|
||||
switch cmd {
|
||||
case control.CmdStatus:
|
||||
return `OK {"version":"v9.9.9","uptime_s":5}`
|
||||
case control.CmdUpdateNow:
|
||||
return "OK v9.9.9 is up to date"
|
||||
default:
|
||||
return "ERR unknown command: " + cmd
|
||||
}
|
||||
}, "", "")
|
||||
return httptest.NewServer(h)
|
||||
}
|
||||
|
||||
func TestUIPage(t *testing.T) {
|
||||
srv := testUIHandler()
|
||||
defer srv.Close()
|
||||
resp, err := srv.Client().Get(srv.URL + "/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body := make([]byte, 1024*64)
|
||||
n, _ := resp.Body.Read(body)
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("status %d", resp.StatusCode)
|
||||
}
|
||||
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/html") {
|
||||
t.Errorf("Content-Type %q", ct)
|
||||
}
|
||||
if !strings.Contains(string(body[:n]), "gpu-turnstile") {
|
||||
t.Error("page does not mention gpu-turnstile")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIStatus(t *testing.T) {
|
||||
srv := testUIHandler()
|
||||
defer srv.Close()
|
||||
resp, err := srv.Client().Get(srv.URL + "/api/status")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body := make([]byte, 4096)
|
||||
n, _ := resp.Body.Read(body)
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("status %d", resp.StatusCode)
|
||||
}
|
||||
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "application/json") {
|
||||
t.Errorf("Content-Type %q", ct)
|
||||
}
|
||||
if got := strings.TrimSpace(string(body[:n])); got != `{"version":"v9.9.9","uptime_s":5}` {
|
||||
t.Errorf("body %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIActions(t *testing.T) {
|
||||
srv := testUIHandler()
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := srv.Client().Post(srv.URL+"/api/action/update-now", "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body := make([]byte, 4096)
|
||||
n, _ := resp.Body.Read(body)
|
||||
if resp.StatusCode != 200 || string(body[:n]) != "v9.9.9 is up to date" {
|
||||
t.Errorf("update-now: %d %q", resp.StatusCode, body[:n])
|
||||
}
|
||||
|
||||
// The handler answers ERR for reload-env: mapped to 409 with the
|
||||
// message stripped.
|
||||
resp2, err := srv.Client().Post(srv.URL+"/api/action/reload-env", "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
n, _ = resp2.Body.Read(body)
|
||||
if resp2.StatusCode != 409 || !strings.HasPrefix(string(body[:n]), "unknown command") {
|
||||
t.Errorf("reload-env: %d %q", resp2.StatusCode, body[:n])
|
||||
}
|
||||
|
||||
// Unknown actions are not routed to the handler.
|
||||
resp3, err := srv.Client().Post(srv.URL+"/api/action/reboot", "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp3.Body.Close()
|
||||
if resp3.StatusCode != 404 {
|
||||
t.Errorf("bogus action: %d, want 404", resp3.StatusCode)
|
||||
}
|
||||
|
||||
// GET on an action path falls through to the root handler's guard,
|
||||
// which only serves "/" itself.
|
||||
resp4, err := srv.Client().Get(srv.URL + "/api/action/update-now")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp4.Body.Close()
|
||||
if resp4.StatusCode != 404 {
|
||||
t.Errorf("GET action: %d, want 404", resp4.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIBasicAuth(t *testing.T) {
|
||||
h := uiHandler(func(cmd string) string { return `OK {"version":"v9.9.9"}` }, "admin", "s3cret")
|
||||
srv := httptest.NewServer(h)
|
||||
defer srv.Close()
|
||||
|
||||
// No credentials: 401 with the auth challenge on page and API alike.
|
||||
for _, path := range []string{"/", "/api/status"} {
|
||||
resp, err := srv.Client().Get(srv.URL + path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 401 {
|
||||
t.Errorf("GET %s without auth: %d, want 401", path, resp.StatusCode)
|
||||
}
|
||||
if resp.Header.Get("WWW-Authenticate") == "" {
|
||||
t.Errorf("GET %s: no WWW-Authenticate header", path)
|
||||
}
|
||||
}
|
||||
|
||||
// Wrong credentials: 401. Right credentials: 200.
|
||||
req, _ := http.NewRequest("GET", srv.URL+"/api/status", nil)
|
||||
req.SetBasicAuth("admin", "wrong")
|
||||
resp, err := srv.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 401 {
|
||||
t.Errorf("wrong password: %d, want 401", resp.StatusCode)
|
||||
}
|
||||
|
||||
req, _ = http.NewRequest("GET", srv.URL+"/api/status", nil)
|
||||
req.SetBasicAuth("admin", "s3cret")
|
||||
resp, err = srv.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body := make([]byte, 4096)
|
||||
n, _ := resp.Body.Read(body)
|
||||
if resp.StatusCode != 200 || strings.TrimSpace(string(body[:n])) != `{"version":"v9.9.9"}` {
|
||||
t.Errorf("with auth: %d %q", resp.StatusCode, body[:n])
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsLoopbackHost(t *testing.T) {
|
||||
for host, want := range map[string]bool{
|
||||
"127.0.0.1": true, "localhost": true, "::1": true,
|
||||
"": false, "0.0.0.0": false, "192.168.1.10": false, "example.com": false,
|
||||
} {
|
||||
if got := isLoopbackHost(host); got != want {
|
||||
t.Errorf("isLoopbackHost(%q) = %v, want %v", host, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIURL(t *testing.T) {
|
||||
for addr, want := range map[string]string{
|
||||
"": "",
|
||||
"127.0.0.1:7860": "http://127.0.0.1:7860",
|
||||
":7860": "http://127.0.0.1:7860",
|
||||
"192.168.1.5:9000": "http://192.168.1.5:9000",
|
||||
} {
|
||||
if got := uiURL(addr); got != want {
|
||||
t.Errorf("uiURL(%q) = %q, want %q", addr, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
# ComfyUI --listen 0.0.0.0 --port 8189).
|
||||
services:
|
||||
gpu-turnstile:
|
||||
image: git.rambossek.at/public/gpu-turnstile:v0.3.0
|
||||
image: git.rambossek.at/public/gpu-turnstile:v0.3.2
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Each consumer is enabled by setting its URL; leave one unset to
|
||||
|
||||
@@ -19,6 +19,9 @@ import (
|
||||
type Config struct {
|
||||
ListenOllama string `env:"LISTEN_OLLAMA"`
|
||||
ListenComfy string `env:"LISTEN_COMFY"`
|
||||
ListenUI string `env:"LISTEN_UI"`
|
||||
UIUser string `env:"UI_USER"`
|
||||
UIPass string `env:"UI_PASS"`
|
||||
OllamaURL string `env:"OLLAMA_URL"`
|
||||
ComfyURL string `env:"COMFY_URL"`
|
||||
UnloadTimeout time.Duration `env:"UNLOAD_TIMEOUT"`
|
||||
@@ -196,6 +199,9 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
}{
|
||||
{"LISTEN_OLLAMA", &cfg.ListenOllama},
|
||||
{"LISTEN_COMFY", &cfg.ListenComfy},
|
||||
{"LISTEN_UI", &cfg.ListenUI},
|
||||
{"UI_USER", &cfg.UIUser},
|
||||
{"UI_PASS", &cfg.UIPass},
|
||||
{"OLLAMA_URL", &cfg.OllamaURL},
|
||||
{"COMFY_URL", &cfg.ComfyURL},
|
||||
{"WARM_MODEL", &cfg.WarmModel},
|
||||
@@ -318,6 +324,12 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
default:
|
||||
return cfg, fmt.Errorf("LOG_FORMAT: must be \"text\" or \"json\"")
|
||||
}
|
||||
if (cfg.UIUser == "") != (cfg.UIPass == "") {
|
||||
return cfg, fmt.Errorf("UI_USER and UI_PASS must be set together (both empty = no auth)")
|
||||
}
|
||||
if cfg.UIUser != "" && cfg.ListenUI == "" {
|
||||
return cfg, fmt.Errorf("UI_USER/UI_PASS have no effect without LISTEN_UI")
|
||||
}
|
||||
if cfg.ComfyCmd != "" && cfg.ComfyURL == "" {
|
||||
return cfg, fmt.Errorf("COMFY_CMD requires COMFY_URL to be set (the proxy needs somewhere to forward)")
|
||||
}
|
||||
|
||||
@@ -97,6 +97,30 @@ func TestComfyCmdRequiresURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIAuthPairing(t *testing.T) {
|
||||
env := func(set map[string]string) func(string) string {
|
||||
return func(k string) string { return set[k] }
|
||||
}
|
||||
// Only one of UI_USER/UI_PASS: error.
|
||||
_, err := Load(env(map[string]string{"OLLAMA_URL": "http://x", "LISTEN_UI": "127.0.0.1:7860", "UI_USER": "admin"}))
|
||||
if err == nil || !strings.Contains(err.Error(), "UI_USER and UI_PASS must be set together") {
|
||||
t.Fatalf("err = %v, want pairing error", err)
|
||||
}
|
||||
// Auth without LISTEN_UI: error.
|
||||
_, err = Load(env(map[string]string{"OLLAMA_URL": "http://x", "UI_USER": "admin", "UI_PASS": "x"}))
|
||||
if err == nil || !strings.Contains(err.Error(), "no effect without LISTEN_UI") {
|
||||
t.Fatalf("err = %v, want LISTEN_UI error", err)
|
||||
}
|
||||
// Both with LISTEN_UI: loads.
|
||||
cfg, err := Load(env(map[string]string{"OLLAMA_URL": "http://x", "LISTEN_UI": "127.0.0.1:7860", "UI_USER": "admin", "UI_PASS": "x"}))
|
||||
if err != nil {
|
||||
t.Fatalf("auth pair with LISTEN_UI must load: %v", err)
|
||||
}
|
||||
if cfg.UIUser != "admin" || cfg.UIPass != "x" {
|
||||
t.Errorf("got %q/%q", cfg.UIUser, cfg.UIPass)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvFile(t *testing.T) {
|
||||
input := `# comment
|
||||
OLLAMA_URL=http://host:11435
|
||||
|
||||
@@ -25,6 +25,9 @@ func sampleEntries(logFile string) []sampleEntry {
|
||||
return []sampleEntry{
|
||||
{"LISTEN_OLLAMA", ":11434", "Listen address for Ollama-compatible clients (gpu-turnstile poses as Ollama here)", false},
|
||||
{"LISTEN_COMFY", ":8188", "Listen address for ComfyUI clients (gpu-turnstile poses as ComfyUI here)", false},
|
||||
{"LISTEN_UI", "127.0.0.1:7860", "Web UI listen address (host:port): live status like --monitor, with update/reload buttons (default: empty = disabled; bind 127.0.0.1 unless you set auth)", false},
|
||||
{"UI_USER", "", "HTTP basic auth for the web UI (UI_USER and UI_PASS must be set together; both empty = no auth)", false},
|
||||
{"UI_PASS", "", "See UI_USER", false},
|
||||
{"OLLAMA_URL", "http://127.0.0.1:11434", "Ollama upstream URL; setting it enables the Ollama consumer (default: empty = disabled)", false},
|
||||
{"COMFY_URL", "http://127.0.0.1:8188", "ComfyUI upstream URL; setting it enables the ComfyUI consumer (default: empty = disabled)", false},
|
||||
{"WARM_MODEL", "", "Optional model to reload after an image job (default: empty = none)", false},
|
||||
|
||||
+42
-15
@@ -186,27 +186,54 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
|
||||
return parseComputeApps(string(out))
|
||||
}
|
||||
|
||||
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
|
||||
// Unlike the per-process list this works under WDDM too.
|
||||
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
|
||||
// GPUStats is one nvidia-smi reading of the whole card.
|
||||
type GPUStats struct {
|
||||
UsedMB int
|
||||
TotalMB int
|
||||
TempC int // -1 when nvidia-smi reports N/A
|
||||
FanPct int // -1 when N/A (some cards don't expose the fan)
|
||||
}
|
||||
|
||||
// QueryGPUStats returns VRAM usage, temperature and fan speed via
|
||||
// nvidia-smi. Unlike the per-process list this works under WDDM too.
|
||||
func QueryGPUStats(ctx context.Context) (GPUStats, error) {
|
||||
out, err := exec.CommandContext(ctx, "nvidia-smi",
|
||||
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
|
||||
"--query-gpu=memory.used,memory.total,temperature.gpu,fan.speed", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
return GPUStats{}, err
|
||||
}
|
||||
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
|
||||
return parseGPUStats(string(out))
|
||||
}
|
||||
|
||||
// parseGPUStats parses one "used, total, temp, fan" CSV line (MiB, °C,
|
||||
// percent). The memory fields must be numeric; temperature and fan fall
|
||||
// back to -1 on "N/A" and friends.
|
||||
func parseGPUStats(out string) (GPUStats, error) {
|
||||
fields := strings.Split(strings.TrimSpace(out), ",")
|
||||
if len(fields) != 4 {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(out))
|
||||
}
|
||||
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
|
||||
num := func(s string) (int, error) {
|
||||
return strconv.Atoi(strings.TrimSpace(s))
|
||||
}
|
||||
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
|
||||
optional := func(s string) int {
|
||||
n, err := num(s)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return n
|
||||
}
|
||||
return used, total, nil
|
||||
var st GPUStats
|
||||
var err error
|
||||
if st.UsedMB, err = num(fields[0]); err != nil {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(out))
|
||||
}
|
||||
if st.TotalMB, err = num(fields[1]); err != nil {
|
||||
return GPUStats{}, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(out))
|
||||
}
|
||||
st.TempC = optional(fields[2])
|
||||
st.FanPct = optional(fields[3])
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
|
||||
|
||||
@@ -116,6 +116,31 @@ func TestParseGPUEngineInstance(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGPUStats(t *testing.T) {
|
||||
st, err := parseGPUStats("4300, 16384, 55, 42\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.UsedMB != 4300 || st.TotalMB != 16384 || st.TempC != 55 || st.FanPct != 42 {
|
||||
t.Errorf("got %+v", st)
|
||||
}
|
||||
|
||||
// Cards that don't expose temperature/fan report N/A.
|
||||
st, err = parseGPUStats("1024, 16384, N/A, N/A")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.TempC != -1 || st.FanPct != -1 {
|
||||
t.Errorf("got %+v, want -1 for N/A fields", st)
|
||||
}
|
||||
|
||||
for _, bad := range []string{"", "1, 2", "x, 16384, 55, 42", "1024, x, 55, 42", "1, 2, 3, 4, 5"} {
|
||||
if _, err := parseGPUStats(bad); err == nil {
|
||||
t.Errorf("%q parsed, want failure", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesLive(t *testing.T) {
|
||||
if runtime.GOOS != "windows" && runtime.GOOS != "linux" {
|
||||
t.Skip("no process listing on this platform")
|
||||
|
||||
Reference in New Issue
Block a user