Compare commits

...
3 Commits
Author SHA1 Message Date
mram 4bd5f34ce7 Pin compose example to v0.2.8
ci / test (push) Successful in 15s
ci / docker (push) Successful in 1m6s
ci / release (push) Successful in 15s
2026-09-22 09:10:14 +02:00
mram b9d3f91403 Monitor: hotkeys (q quit, u update now) with footer line; show GPU VRAM usage 2026-09-22 09:10:14 +02:00
mram 98d2d714ca Add --reload-env: service re-reads and validates its config, restarts when idle if it changed 2026-09-22 09:02:25 +02:00
8 changed files with 320 additions and 39 deletions
+136 -33
View File
@@ -16,6 +16,7 @@ import (
"os" "os"
"os/signal" "os/signal"
"path/filepath" "path/filepath"
"reflect"
"runtime" "runtime"
"strings" "strings"
"sync" "sync"
@@ -39,7 +40,8 @@ import (
var version = "dev" var version = "dev"
// exitCodeUpdate tells the service recovery configuration to restart the // exitCodeUpdate tells the service recovery configuration to restart the
// process: a signed update has been staged and the GPU lock is idle. // process: a signed update has been staged or a changed config reload was
// requested, and the GPU lock is idle.
const exitCodeUpdate = 3 const exitCodeUpdate = 3
// exitCodeStaged is returned by an elevated --force-update child when it // exitCodeStaged is returned by an elevated --force-update child when it
@@ -59,7 +61,7 @@ func stdoutIsTerminal() bool {
// --install-service / --remove-service switches, --no-copy, -h/--help, // --install-service / --remove-service switches, --no-copy, -h/--help,
// -v/--version, --force-update and the hidden --elevated-child marker from // -v/--version, --force-update and the hidden --elevated-child marker from
// args. // args.
func parseFlags(args []string) (configPath string, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild bool, rest []string) { func parseFlags(args []string) (configPath string, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild bool, rest []string) {
rest = args[:0] rest = args[:0]
for i := 0; i < len(args); i++ { for i := 0; i < len(args); i++ {
switch { switch {
@@ -82,6 +84,8 @@ func parseFlags(args []string) (configPath string, install, remove, noCopy, help
forceUpdate = true forceUpdate = true
case args[i] == "--update-now" || args[i] == "-update-now": case args[i] == "--update-now" || args[i] == "-update-now":
updateNow = true updateNow = true
case args[i] == "--reload-env" || args[i] == "-reload-env":
reloadEnv = true
case args[i] == "--monitor" || args[i] == "-monitor" || args[i] == "-m": case args[i] == "--monitor" || args[i] == "-monitor" || args[i] == "-m":
monitor = true monitor = true
case args[i] == "--elevated-child": case args[i] == "--elevated-child":
@@ -90,7 +94,7 @@ func parseFlags(args []string) (configPath string, install, remove, noCopy, help
rest = append(rest, args[i]) rest = append(rest, args[i])
} }
} }
return configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild, rest return configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild, rest
} }
// versionLine is printed at the top of every help and error screen. // versionLine is printed at the top of every help and error screen.
@@ -109,6 +113,9 @@ Usage:
(no admin needed when the service runs) (no admin needed when the service runs)
gpu-turnstile --update-now like --force-update, but only gpu-turnstile --update-now like --force-update, but only
through the running service through the running service
gpu-turnstile --reload-env make the service re-read and
validate its config file, then
restart onto it if it changed
gpu-turnstile -m | --monitor live status view (downstreams, gpu-turnstile -m | --monitor live status view (downstreams,
GPU lock, queue); Ctrl+C quits GPU lock, queue); Ctrl+C quits
gpu-turnstile -h | --help this help gpu-turnstile -h | --help this help
@@ -141,12 +148,12 @@ func fatalUsage(format string, args ...any) {
} }
func main() { func main() {
configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild, args := parseFlags(os.Args[1:]) configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild, args := parseFlags(os.Args[1:])
if showVersion { if showVersion {
fmt.Println(version) fmt.Println(version)
return return
} }
bare := configPath == "" && !install && !remove && !forceUpdate && !updateNow && !monitor && !elevatedChild && len(args) == 0 bare := configPath == "" && !install && !remove && !forceUpdate && !updateNow && !reloadEnv && !monitor && !elevatedChild && len(args) == 0
if help || (bare && stdoutIsTerminal()) { if help || (bare && stdoutIsTerminal()) {
// Bare invocation in a terminal (e.g. double-clicked on Windows) // Bare invocation in a terminal (e.g. double-clicked on Windows)
// shows the help instead of starting a proxy window with no visible // shows the help instead of starting a proxy window with no visible
@@ -171,7 +178,9 @@ func main() {
fatalUsage("error: --force-update cannot be combined with --install-service/--remove-service") fatalUsage("error: --force-update cannot be combined with --install-service/--remove-service")
case updateNow && (install || remove || forceUpdate): case updateNow && (install || remove || forceUpdate):
fatalUsage("error: --update-now cannot be combined with other commands") fatalUsage("error: --update-now cannot be combined with other commands")
case monitor && (install || remove || forceUpdate || updateNow): case reloadEnv && (install || remove || forceUpdate || updateNow):
fatalUsage("error: --reload-env cannot be combined with other commands")
case monitor && (install || remove || forceUpdate || updateNow || reloadEnv):
fatalUsage("error: --monitor cannot be combined with other commands") fatalUsage("error: --monitor cannot be combined with other commands")
case install: case install:
os.Exit(serviceCommand(configPath, true, noCopy, elevatedChild)) os.Exit(serviceCommand(configPath, true, noCopy, elevatedChild))
@@ -181,6 +190,8 @@ func main() {
os.Exit(forceUpdateCommand(configPath, elevatedChild)) os.Exit(forceUpdateCommand(configPath, elevatedChild))
case updateNow: case updateNow:
os.Exit(updateNowCommand()) os.Exit(updateNowCommand())
case reloadEnv:
os.Exit(reloadEnvCommand())
case monitor: case monitor:
os.Exit(monitorCommand()) os.Exit(monitorCommand())
} }
@@ -202,7 +213,7 @@ func main() {
syncEnvFile(resolveConfigPath(configPath), cfg.LogFile, log) syncEnvFile(resolveConfigPath(configPath), cfg.LogFile, log)
if service.IsService() { if service.IsService() {
if err := service.Run(func(ctx context.Context) error { return run(ctx, cfg, log, logOut, true) }); err != nil { if err := service.Run(func(ctx context.Context) error { return run(ctx, cfg, log, logOut, true, configPath) }); err != nil {
log.Error("service failed", "err", err) log.Error("service failed", "err", err)
os.Exit(1) os.Exit(1)
} }
@@ -210,7 +221,7 @@ func main() {
} }
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop() defer stop()
if err := run(ctx, cfg, log, logOut, false); err != nil { if err := run(ctx, cfg, log, logOut, false, configPath); err != nil {
log.Error("listener failed", "err", err) log.Error("listener failed", "err", err)
os.Exit(1) os.Exit(1)
} }
@@ -451,6 +462,18 @@ func updateNowCommand() int {
return printControlReply(reply) return printControlReply(reply)
} }
// reloadEnvCommand asks the running service, over the control channel, to
// re-read and validate its config file. The client sends no path — the
// service only ever re-reads its own configured file.
func reloadEnvCommand() int {
reply, err := control.Ask(control.CmdReloadEnv)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n\ngpu-turnstile: no running service to ask\n", versionLine())
return 1
}
return printControlReply(reply)
}
// reportElevatedUpdate prints the parent's summary of an elevated // reportElevatedUpdate prints the parent's summary of an elevated
// --force-update child: exitCodeStaged means the child staged a new binary, // --force-update child: exitCodeStaged means the child staged a new binary,
// 0 means it found nothing to do. to is the tag the parent's own check // 0 means it found nothing to do. to is the tag the parent's own check
@@ -531,7 +554,7 @@ func managedComfyCommand(cfg config.Config) string {
return "" return ""
} }
func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Writer, isService bool) error { func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Writer, isService bool, configPath string) error {
// ComfyUI can run as a managed child — COMFY_CMD verbatim, or the // ComfyUI can run as a managed child — COMFY_CMD verbatim, or the
// standard venv layout derived from COMFY_DIR alone: started on demand // standard venv layout derived from COMFY_DIR alone: started on demand
// by the proxy, stopped after COMFY_IDLE_TIMEOUT idle (and on shutdown) // by the proxy, stopped after COMFY_IDLE_TIMEOUT idle (and on shutdown)
@@ -705,9 +728,11 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
// Foreign GPU holders (games, other ML jobs) — enabled by GAME_PROCS // Foreign GPU holders (games, other ML jobs) — enabled by GAME_PROCS
// and/or GPU_FOREIGN_VRAM_MB — hold the lock externally while they run. // and/or GPU_FOREIGN_VRAM_MB — hold the lock externally while they run.
// gw collects the VRAM reading for the status channel.
gw := &gpuWatch{}
if len(cfg.GameProcs) > 0 || cfg.GPUForeignVRAMMB > 0 { if len(cfg.GameProcs) > 0 || cfg.GPUForeignVRAMMB > 0 {
det := game.New(cfg.GameProcs, cfg.GPUForeignVRAMMB, cfg.GPUIgnoreProcs, log) det := game.New(cfg.GameProcs, cfg.GPUForeignVRAMMB, cfg.GPUIgnoreProcs, log)
go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup) go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup, gw)
} }
// Bind the listeners up front so a port conflict fails fast and the // Bind the listeners up front so a port conflict fails fast and the
@@ -747,6 +772,22 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
var u *update.Updater var u *update.Updater
var exePath string var exePath string
// restartWhenIdle exits with exitCodeUpdate once the GPU lock is idle;
// the service recovery configuration brings the process back. Shared by
// staged updates and config reloads; the once guard makes repeated
// triggers idempotent.
var restartOnce sync.Once
restartWhenIdle := func(reason string) {
log.Warn("restarting once the GPU is idle", "reason", reason)
restartOnce.Do(func() {
go func() {
if waitForIdle(ctx, lk, 24*time.Hour) {
log.Warn("restarting now", "reason", reason)
os.Exit(exitCodeUpdate)
}
}()
})
}
applyStaged := func(to string) {} applyStaged := func(to string) {}
if cfg.AutoUpdate { if cfg.AutoUpdate {
if p, err := os.Executable(); err != nil { if p, err := os.Executable(); err != nil {
@@ -754,34 +795,24 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
} else { } else {
exePath = p exePath = p
u = &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log} u = &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log}
// applyStaged is shared by the hourly loop and the control
// channel; the once guard keeps a second trigger from
// double-waiting on the GPU lock.
var once sync.Once
applyStaged = func(to string) { applyStaged = func(to string) {
if !isService { if !isService {
log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply", "version", to) log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply", "version", to)
return return
} }
log.Warn("auto-update: staged; restarting once the GPU is idle", "version", to) restartWhenIdle("update to " + to)
once.Do(func() {
go func() {
if waitForIdle(ctx, lk, 24*time.Hour) {
log.Warn("auto-update: restarting to apply update")
os.Exit(exitCodeUpdate)
}
}()
})
} }
go updateLoop(ctx, cfg.UpdateInterval, log, u, exePath, applyStaged) go updateLoop(ctx, cfg.UpdateInterval, log, u, exePath, applyStaged)
} }
} }
// The control channel (status for --monitor, update-now trigger) is // The control channel (status for --monitor, update-now and reload-env
// served whenever running as a service, independent of AUTO_UPDATE. // triggers) is served whenever running as a service, independent of
// AUTO_UPDATE.
if isService { if isService {
serveControl(ctx, log, u, exePath, applyStaged, serveControl(ctx, log, u, exePath, applyStaged,
statusProvider(cfg, lk, comfySup, health, started)) statusProvider(cfg, lk, comfySup, health, started, gw),
reloadHandler(cfg, configPath, restartWhenIdle))
} }
select { select {
@@ -805,12 +836,34 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
// (idle); health checks skip it instead of logging an outage. // (idle); health checks skip it instead of logging an outage.
var errManagedDown = errors.New("managed upstream intentionally stopped") var errManagedDown = errors.New("managed upstream intentionally stopped")
// gpuWatch records the latest VRAM reading from the game detector's poll
// loop, for the status channel. Known stays false when game detection is
// not configured (no nvidia-smi polling happens then).
type gpuWatch struct {
mu sync.Mutex
usedMB int
total int
known bool
}
func (g *gpuWatch) set(used, total int) {
g.mu.Lock()
g.usedMB, g.total, g.known = used, total, true
g.mu.Unlock()
}
func (g *gpuWatch) get() (used, total int, known bool) {
g.mu.Lock()
defer g.mu.Unlock()
return g.usedMB, g.total, g.known
}
// gameLoop polls for foreign GPU holders (a game, another ML job). While one // gameLoop polls for foreign GPU holders (a game, another ML job). While one
// is detected it holds the lock externally so new LLM and image requests // is detected it holds the lock externally so new LLM and image requests
// wait (or are rejected per LLM_BUSY_MODE), and — once in-flight work has // wait (or are rejected per LLM_BUSY_MODE), and — once in-flight work has
// drained — frees VRAM for it: the managed ComfyUI is stopped and Ollama's // drained — frees VRAM for it: the managed ComfyUI is stopped and Ollama's
// resident models are unloaded. // resident models are unloaded.
func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process) { func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process, gw *gpuWatch) {
ticker := time.NewTicker(cfg.GamePollInterval) ticker := time.NewTicker(cfg.GamePollInterval)
defer ticker.Stop() defer ticker.Stop()
held, freed := false, false held, freed := false, false
@@ -824,6 +877,9 @@ func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *gam
if err != nil && ctx.Err() == nil { if err != nil && ctx.Err() == nil {
log.Warn("game detection failed", "err", err) log.Warn("game detection failed", "err", err)
} }
if used, total, verr := game.QueryVRAMMB(ctx); verr == nil {
gw.set(used, total)
}
switch { switch {
case len(holders) > 0 && !held: case len(holders) > 0 && !held:
held = true held = true
@@ -949,17 +1005,20 @@ func updateLoop(ctx context.Context, interval time.Duration, log *slog.Logger, u
// serveControl opens the local control channel (named pipe on Windows, // serveControl opens the local control channel (named pipe on Windows,
// unix socket on Linux) so unprivileged local users can query status // unix socket on Linux) so unprivileged local users can query status
// (--monitor) and trigger an update check (--force-update/--update-now) // (--monitor), trigger an update check (--force-update/--update-now) and
// without admin rights. The update payload is signature-verified regardless // poke a config reload (--reload-env) without admin rights. The update
// of who asks; triggers are rate-limited to one per minute so the channel // payload is signature-verified regardless of who asks; update triggers
// cannot be used to spam restarts. u is nil when AUTO_UPDATE=false. // are rate-limited to one per minute so the channel cannot be used to spam
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string), status func() string) { // restarts. u is nil when AUTO_UPDATE=false.
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string), status func() string, reload func() string) {
var mu sync.Mutex var mu sync.Mutex
var lastTrigger time.Time var lastTrigger time.Time
h := func(cmd string) string { h := func(cmd string) string {
switch cmd { switch cmd {
case control.CmdStatus: case control.CmdStatus:
return "OK " + status() return "OK " + status()
case control.CmdReloadEnv:
return reload()
case control.CmdUpdateNow: case control.CmdUpdateNow:
default: default:
return "ERR unknown command: " + cmd return "ERR unknown command: " + cmd
@@ -1038,18 +1097,62 @@ type statusSnapshot struct {
UptimeS int64 `json:"uptime_s"` UptimeS int64 `json:"uptime_s"`
Downstreams []statusDownstream `json:"downstreams"` Downstreams []statusDownstream `json:"downstreams"`
Lock statusLock `json:"lock"` Lock statusLock `json:"lock"`
// GPU carries the latest VRAM reading; Known is false when game
// detection (and with it nvidia-smi polling) is not configured.
GPU statusGPU `json:"gpu"`
// MonitorNote is set client-side (never over the wire) when the // MonitorNote is set client-side (never over the wire) when the
// monitor's own binary differs from the service's version. // monitor's own binary differs from the service's version.
MonitorNote string `json:"-"` MonitorNote string `json:"-"`
} }
type statusGPU struct {
UsedMB int `json:"used_mb"`
TotalMB int `json:"total_mb"`
Known bool `json:"known"`
}
// reloadHandler re-reads and validates the service's config file for
// CmdReloadEnv. An invalid config is reported and the service keeps running
// untouched; a valid, changed config triggers a GPU-idle-gated restart onto
// it (same mechanism as staged updates); unchanged is a no-op.
func reloadHandler(current config.Config, configPath string, restartWhenIdle func(reason string)) func() string {
return func() string {
ncfg, err := loadMergedConfig(configPath)
if err != nil {
return "ERR config invalid: " + err.Error()
}
changed := diffConfig(current, ncfg)
if len(changed) == 0 {
return "OK config unchanged"
}
restartWhenIdle("config reload (" + strings.Join(changed, ", ") + ")")
return "OK config valid; restarting once the GPU is idle (changed: " + strings.Join(changed, ", ") + ")"
}
}
// diffConfig lists the names of fields whose values differ between two
// configs.
func diffConfig(a, b config.Config) []string {
va, vb := reflect.ValueOf(a), reflect.ValueOf(b)
t := va.Type()
var out []string
for i := 0; i < t.NumField(); i++ {
if !reflect.DeepEqual(va.Field(i).Interface(), vb.Field(i).Interface()) {
out = append(out, t.Field(i).Name)
}
}
return out
}
// statusProvider assembles the one-line JSON snapshot for CmdStatus. // statusProvider assembles the one-line JSON snapshot for CmdStatus.
func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time) func() string { func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time, gw *gpuWatch) func() string {
return func() string { return func() string {
snap := statusSnapshot{ snap := statusSnapshot{
Version: version, Version: version,
UptimeS: int64(time.Since(started).Seconds()), UptimeS: int64(time.Since(started).Seconds()),
} }
used, total, known := gw.get()
snap.GPU = statusGPU{UsedMB: used, TotalMB: total, Known: known}
if cfg.OllamaURL != "" { if cfg.OllamaURL != "" {
snap.Downstreams = append(snap.Downstreams, statusDownstream{ snap.Downstreams = append(snap.Downstreams, statusDownstream{
Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"), Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"),
+103 -5
View File
@@ -21,17 +21,26 @@ const (
cCyan = "\x1b[36m" cCyan = "\x1b[36m"
) )
// hotkeysLine is the monitor's footer.
const hotkeysLine = " " + cDim + "q quit · u update now" + cReset + "\x1b[K\n"
// monitorCommand renders a live status view of the running service, // monitorCommand renders a live status view of the running service,
// refreshed every second from the control channel. When the service // refreshed every second from the control channel. When the service
// reports a different version and the executable on disk changed (the // reports a different version and the executable on disk changed (the
// updater replaced it), the monitor restarts itself onto the new binary. // updater replaced it), the monitor restarts itself onto the new binary.
// Ctrl+C quits. // Hotkeys: q quits, u triggers an update check on the service.
func monitorCommand() int { func monitorCommand() int {
if !stdoutIsTerminal() { if !stdoutIsTerminal() {
fmt.Fprintln(os.Stderr, "gpu-turnstile: --monitor needs an interactive terminal") fmt.Fprintln(os.Stderr, "gpu-turnstile: --monitor needs an interactive terminal")
return 1 return 1
} }
enableVirtualTerminal() enableVirtualTerminal()
restore := enableRawKeys()
defer func() {
if restore != nil {
restore()
}
}()
fmt.Print("\x1b[2J") // clear once; frames then redraw in place fmt.Print("\x1b[2J") // clear once; frames then redraw in place
defer fmt.Print(cReset + "\n") defer fmt.Print(cReset + "\n")
exe, _ := os.Executable() exe, _ := os.Executable()
@@ -39,7 +48,18 @@ func monitorCommand() int {
if st, err := os.Stat(exe); err == nil { if st, err := os.Stat(exe); err == nil {
exeStamp = st.ModTime() exeStamp = st.ModTime()
} }
for {
keys := make(chan byte, 8)
go readKeys(keys)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
var note string
var noteAt time.Time
noteCh := make(chan string, 1)
updatePending := false
poll := func() string {
frame := renderWaiting() frame := renderWaiting()
if reply, err := control.Ask(control.CmdStatus); err == nil { if reply, err := control.Ask(control.CmdStatus); err == nil {
if msg, ok := strings.CutPrefix(reply, "OK "); ok { if msg, ok := strings.CutPrefix(reply, "OK "); ok {
@@ -50,16 +70,75 @@ func monitorCommand() int {
fmt.Print("\x1b[2J\x1b[H") fmt.Print("\x1b[2J\x1b[H")
fmt.Printf("gpu-turnstile: service updated to %s — restarting the monitor\n", snap.Version) fmt.Printf("gpu-turnstile: service updated to %s — restarting the monitor\n", snap.Version)
restartSelf(exe, "--monitor") restartSelf(exe, "--monitor")
return 0 return "" // re-execed; this process exits below
} }
snap.MonitorNote = fmt.Sprintf("note: the service runs %s, this monitor is %s", snap.Version, version) snap.MonitorNote = fmt.Sprintf("note: the service runs %s, this monitor is %s", snap.Version, version)
} }
if note != "" {
snap.MonitorNote = note
}
frame = renderMonitor(snap, termWidth()) frame = renderMonitor(snap, termWidth())
} }
} }
} }
return frame
}
for {
frame := poll()
if frame == "" {
return 0 // restartSelf fired
}
fmt.Print("\x1b[H" + frame + "\x1b[J") // home, frame, clear below fmt.Print("\x1b[H" + frame + "\x1b[J") // home, frame, clear below
time.Sleep(time.Second) select {
case <-ticker.C:
if note != "" && time.Since(noteAt) > 15*time.Second {
note = ""
}
case k, ok := <-keys:
if !ok {
keys = nil
continue
}
switch k {
case 'q', 'Q', 3: // q or Ctrl+C (raw mode delivers it as a byte)
return 0
case 'u', 'U':
if !updatePending {
updatePending = true
note, noteAt = "checking for updates…", time.Now()
go func() {
reply, err := control.Ask(control.CmdUpdateNow)
if err != nil {
noteCh <- "update: no answer from the service"
return
}
msg := strings.TrimPrefix(reply, "OK ")
msg = strings.TrimPrefix(msg, "ERR ")
noteCh <- "update: " + msg
}()
}
}
case n := <-noteCh:
updatePending = false
note, noteAt = n, time.Now()
}
}
}
// readKeys reads single keypresses from stdin (raw mode was enabled by the
// caller) and delivers them until stdin fails.
func readKeys(keys chan<- byte) {
defer close(keys)
buf := make([]byte, 1)
for {
n, err := os.Stdin.Read(buf)
if n > 0 {
keys <- buf[0]
}
if err != nil {
return
}
} }
} }
@@ -82,7 +161,7 @@ func restartSelf(exe string, args ...string) {
} }
func renderWaiting() string { func renderWaiting() string {
return cDim + " gpu-turnstile — waiting for a running service…" + cReset + "\x1b[K\n" return cDim + " gpu-turnstile — waiting for a running service…" + cReset + "\x1b[K\n\x1b[K\n" + hotkeysLine
} }
// renderMonitor draws one full frame. Each line ends with \x1b[K (clear to // renderMonitor draws one full frame. Each line ends with \x1b[K (clear to
@@ -117,12 +196,31 @@ func renderMonitor(snap statusSnapshot, width int) string {
b.WriteString(fmt.Sprintf(" Queue: %s%d image job(s) waiting%s\x1b[K\n", b.WriteString(fmt.Sprintf(" Queue: %s%d image job(s) waiting%s\x1b[K\n",
cYellow, snap.Lock.ImageQueue, cReset)) cYellow, snap.Lock.ImageQueue, cReset))
} }
if snap.GPU.Known {
b.WriteString(" GPU: " + renderVRAM(snap.GPU.UsedMB, snap.GPU.TotalMB) + "\x1b[K\n")
}
if snap.MonitorNote != "" { if snap.MonitorNote != "" {
b.WriteString(" " + cYellow + snap.MonitorNote + cReset + "\x1b[K\n") b.WriteString(" " + cYellow + snap.MonitorNote + cReset + "\x1b[K\n")
} }
b.WriteString("\x1b[K\n")
b.WriteString(hotkeysLine)
return b.String() return b.String()
} }
// renderVRAM renders "4.2 / 16.0 GiB used" (or MiB below 1 GiB).
func renderVRAM(used, total int) string {
format := func(mb int) string {
if mb >= 1024 {
return fmt.Sprintf("%.1f GiB", float64(mb)/1024)
}
return fmt.Sprintf("%d MiB", mb)
}
if total > 0 {
return format(used) + " / " + format(total) + " used"
}
return format(used) + " used"
}
// printableLen counts characters without ANSI escapes (ASCII-only content). // printableLen counts characters without ANSI escapes (ASCII-only content).
func printableLen(s string) int { return len(s) } func printableLen(s string) int { return len(s) }
+15
View File
@@ -1,6 +1,7 @@
package main package main
import ( import (
"gpu-turnstile/internal/config"
"strings" "strings"
"testing" "testing"
) )
@@ -39,3 +40,17 @@ func TestFmtDur(t *testing.T) {
} }
} }
} }
func TestDiffConfig(t *testing.T) {
a := config.Defaults()
b := a
if got := diffConfig(a, b); len(got) != 0 {
t.Fatalf("identical configs: got %v", got)
}
b.LogLevel = -4
b.GameProcs = []string{"game.exe"}
got := diffConfig(a, b)
if len(got) != 2 || got[0] != "GameProcs" || got[1] != "LogLevel" {
t.Fatalf("got %v, want [GameProcs LogLevel]", got)
}
}
+18
View File
@@ -19,3 +19,21 @@ func termWidth() int {
} }
return int(ws.Col) return int(ws.Col)
} }
// enableRawKeys switches the terminal to per-keypress mode (ICANON and ECHO
// off) and returns the restore function, nil when stdin is not a terminal.
func enableRawKeys() func() {
fd := int(os.Stdin.Fd())
term, err := unix.IoctlGetTermios(fd, unix.TCGETS)
if err != nil {
return nil
}
raw := *term
raw.Lflag &^= unix.ICANON | unix.ECHO
raw.Cc[unix.VMIN] = 1
raw.Cc[unix.VTIME] = 0
if err := unix.IoctlSetTermios(fd, unix.TCSETS, &raw); err != nil {
return nil
}
return func() { unix.IoctlSetTermios(fd, unix.TCSETS, term) } //nolint:errcheck
}
+20
View File
@@ -27,3 +27,23 @@ func termWidth() int {
} }
return int(info.Window.Right-info.Window.Left) + 1 return int(info.Window.Right-info.Window.Left) + 1
} }
// enableRawKeys puts the console's stdin into per-keypress mode (no line
// buffering, no echo) and returns the restore function. When stdin is not a
// real console (mintty/Git Bash pipes) it returns nil: ptys already deliver
// keystrokes immediately.
func enableRawKeys() func() {
h := windows.Handle(os.Stdin.Fd())
var mode uint32
if err := windows.GetConsoleMode(h, &mode); err != nil {
return nil
}
const (
enableLineInput = 0x0002
enableEchoInput = 0x0004
)
if err := windows.SetConsoleMode(h, mode&^(enableLineInput|enableEchoInput)); err != nil {
return nil
}
return func() { windows.SetConsoleMode(h, mode) } //nolint:errcheck
}
+1 -1
View File
@@ -6,7 +6,7 @@
# ComfyUI --listen 0.0.0.0 --port 8189). # ComfyUI --listen 0.0.0.0 --port 8189).
services: services:
gpu-turnstile: gpu-turnstile:
image: git.rambossek.at/public/gpu-turnstile:v0.2.7 image: git.rambossek.at/public/gpu-turnstile:v0.2.8
restart: unless-stopped restart: unless-stopped
environment: environment:
# Each consumer is enabled by setting its URL; leave one unset to # Each consumer is enabled by setting its URL; leave one unset to
+4
View File
@@ -26,6 +26,10 @@ const CmdUpdateNow = "update-now"
// CmdStatus asks for a one-line JSON status snapshot (monitor mode). // CmdStatus asks for a one-line JSON status snapshot (monitor mode).
const CmdStatus = "status" const CmdStatus = "status"
// CmdReloadEnv asks the service to re-read and validate its config file,
// and to restart onto it (once the GPU is idle) when it changed.
const CmdReloadEnv = "reload-env"
// ErrUnavailable means no running service offers the control channel. // ErrUnavailable means no running service offers the control channel.
var ErrUnavailable = errors.New("control channel unavailable") var ErrUnavailable = errors.New("control channel unavailable")
+23
View File
@@ -132,6 +132,29 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
return parseComputeApps(string(out)) return parseComputeApps(string(out))
} }
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
// Unlike the per-process list this works under WDDM too.
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
out, err := exec.CommandContext(ctx, "nvidia-smi",
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
if err != nil {
return 0, 0, err
}
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
if !ok {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
}
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
if err != nil {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
}
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
if err != nil {
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
}
return used, total, nil
}
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB // parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
// units). Unsupported rows ("N/A" on WDDM) are skipped. // units). Unsupported rows ("N/A" on WDDM) are skipped.
func parseComputeApps(out string) ([]computeApp, error) { func parseComputeApps(out string) ([]computeApp, error) {