Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4bd5f34ce7 | ||
|
|
b9d3f91403 | ||
|
|
98d2d714ca |
+136
-33
@@ -16,6 +16,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -39,7 +40,8 @@ import (
|
||||
var version = "dev"
|
||||
|
||||
// exitCodeUpdate tells the service recovery configuration to restart the
|
||||
// process: a signed update has been staged and the GPU lock is idle.
|
||||
// process: a signed update has been staged or a changed config reload was
|
||||
// requested, and the GPU lock is idle.
|
||||
const exitCodeUpdate = 3
|
||||
|
||||
// exitCodeStaged is returned by an elevated --force-update child when it
|
||||
@@ -59,7 +61,7 @@ func stdoutIsTerminal() bool {
|
||||
// --install-service / --remove-service switches, --no-copy, -h/--help,
|
||||
// -v/--version, --force-update and the hidden --elevated-child marker from
|
||||
// args.
|
||||
func parseFlags(args []string) (configPath string, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild bool, rest []string) {
|
||||
func parseFlags(args []string) (configPath string, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild bool, rest []string) {
|
||||
rest = args[:0]
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch {
|
||||
@@ -82,6 +84,8 @@ func parseFlags(args []string) (configPath string, install, remove, noCopy, help
|
||||
forceUpdate = true
|
||||
case args[i] == "--update-now" || args[i] == "-update-now":
|
||||
updateNow = true
|
||||
case args[i] == "--reload-env" || args[i] == "-reload-env":
|
||||
reloadEnv = true
|
||||
case args[i] == "--monitor" || args[i] == "-monitor" || args[i] == "-m":
|
||||
monitor = true
|
||||
case args[i] == "--elevated-child":
|
||||
@@ -90,7 +94,7 @@ func parseFlags(args []string) (configPath string, install, remove, noCopy, help
|
||||
rest = append(rest, args[i])
|
||||
}
|
||||
}
|
||||
return configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild, rest
|
||||
return configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild, rest
|
||||
}
|
||||
|
||||
// versionLine is printed at the top of every help and error screen.
|
||||
@@ -109,6 +113,9 @@ Usage:
|
||||
(no admin needed when the service runs)
|
||||
gpu-turnstile --update-now like --force-update, but only
|
||||
through the running service
|
||||
gpu-turnstile --reload-env make the service re-read and
|
||||
validate its config file, then
|
||||
restart onto it if it changed
|
||||
gpu-turnstile -m | --monitor live status view (downstreams,
|
||||
GPU lock, queue); Ctrl+C quits
|
||||
gpu-turnstile -h | --help this help
|
||||
@@ -141,12 +148,12 @@ func fatalUsage(format string, args ...any) {
|
||||
}
|
||||
|
||||
func main() {
|
||||
configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild, args := parseFlags(os.Args[1:])
|
||||
configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild, args := parseFlags(os.Args[1:])
|
||||
if showVersion {
|
||||
fmt.Println(version)
|
||||
return
|
||||
}
|
||||
bare := configPath == "" && !install && !remove && !forceUpdate && !updateNow && !monitor && !elevatedChild && len(args) == 0
|
||||
bare := configPath == "" && !install && !remove && !forceUpdate && !updateNow && !reloadEnv && !monitor && !elevatedChild && len(args) == 0
|
||||
if help || (bare && stdoutIsTerminal()) {
|
||||
// Bare invocation in a terminal (e.g. double-clicked on Windows)
|
||||
// shows the help instead of starting a proxy window with no visible
|
||||
@@ -171,7 +178,9 @@ func main() {
|
||||
fatalUsage("error: --force-update cannot be combined with --install-service/--remove-service")
|
||||
case updateNow && (install || remove || forceUpdate):
|
||||
fatalUsage("error: --update-now cannot be combined with other commands")
|
||||
case monitor && (install || remove || forceUpdate || updateNow):
|
||||
case reloadEnv && (install || remove || forceUpdate || updateNow):
|
||||
fatalUsage("error: --reload-env cannot be combined with other commands")
|
||||
case monitor && (install || remove || forceUpdate || updateNow || reloadEnv):
|
||||
fatalUsage("error: --monitor cannot be combined with other commands")
|
||||
case install:
|
||||
os.Exit(serviceCommand(configPath, true, noCopy, elevatedChild))
|
||||
@@ -181,6 +190,8 @@ func main() {
|
||||
os.Exit(forceUpdateCommand(configPath, elevatedChild))
|
||||
case updateNow:
|
||||
os.Exit(updateNowCommand())
|
||||
case reloadEnv:
|
||||
os.Exit(reloadEnvCommand())
|
||||
case monitor:
|
||||
os.Exit(monitorCommand())
|
||||
}
|
||||
@@ -202,7 +213,7 @@ func main() {
|
||||
syncEnvFile(resolveConfigPath(configPath), cfg.LogFile, log)
|
||||
|
||||
if service.IsService() {
|
||||
if err := service.Run(func(ctx context.Context) error { return run(ctx, cfg, log, logOut, true) }); err != nil {
|
||||
if err := service.Run(func(ctx context.Context) error { return run(ctx, cfg, log, logOut, true, configPath) }); err != nil {
|
||||
log.Error("service failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -210,7 +221,7 @@ func main() {
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, cfg, log, logOut, false); err != nil {
|
||||
if err := run(ctx, cfg, log, logOut, false, configPath); err != nil {
|
||||
log.Error("listener failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -451,6 +462,18 @@ func updateNowCommand() int {
|
||||
return printControlReply(reply)
|
||||
}
|
||||
|
||||
// reloadEnvCommand asks the running service, over the control channel, to
|
||||
// re-read and validate its config file. The client sends no path — the
|
||||
// service only ever re-reads its own configured file.
|
||||
func reloadEnvCommand() int {
|
||||
reply, err := control.Ask(control.CmdReloadEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s\n\ngpu-turnstile: no running service to ask\n", versionLine())
|
||||
return 1
|
||||
}
|
||||
return printControlReply(reply)
|
||||
}
|
||||
|
||||
// reportElevatedUpdate prints the parent's summary of an elevated
|
||||
// --force-update child: exitCodeStaged means the child staged a new binary,
|
||||
// 0 means it found nothing to do. to is the tag the parent's own check
|
||||
@@ -531,7 +554,7 @@ func managedComfyCommand(cfg config.Config) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Writer, isService bool) error {
|
||||
func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Writer, isService bool, configPath string) error {
|
||||
// ComfyUI can run as a managed child — COMFY_CMD verbatim, or the
|
||||
// standard venv layout derived from COMFY_DIR alone: started on demand
|
||||
// by the proxy, stopped after COMFY_IDLE_TIMEOUT idle (and on shutdown)
|
||||
@@ -705,9 +728,11 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
||||
|
||||
// Foreign GPU holders (games, other ML jobs) — enabled by GAME_PROCS
|
||||
// and/or GPU_FOREIGN_VRAM_MB — hold the lock externally while they run.
|
||||
// gw collects the VRAM reading for the status channel.
|
||||
gw := &gpuWatch{}
|
||||
if len(cfg.GameProcs) > 0 || cfg.GPUForeignVRAMMB > 0 {
|
||||
det := game.New(cfg.GameProcs, cfg.GPUForeignVRAMMB, cfg.GPUIgnoreProcs, log)
|
||||
go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup)
|
||||
go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup, gw)
|
||||
}
|
||||
|
||||
// Bind the listeners up front so a port conflict fails fast and the
|
||||
@@ -747,6 +772,22 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
||||
|
||||
var u *update.Updater
|
||||
var exePath string
|
||||
// restartWhenIdle exits with exitCodeUpdate once the GPU lock is idle;
|
||||
// the service recovery configuration brings the process back. Shared by
|
||||
// staged updates and config reloads; the once guard makes repeated
|
||||
// triggers idempotent.
|
||||
var restartOnce sync.Once
|
||||
restartWhenIdle := func(reason string) {
|
||||
log.Warn("restarting once the GPU is idle", "reason", reason)
|
||||
restartOnce.Do(func() {
|
||||
go func() {
|
||||
if waitForIdle(ctx, lk, 24*time.Hour) {
|
||||
log.Warn("restarting now", "reason", reason)
|
||||
os.Exit(exitCodeUpdate)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
applyStaged := func(to string) {}
|
||||
if cfg.AutoUpdate {
|
||||
if p, err := os.Executable(); err != nil {
|
||||
@@ -754,34 +795,24 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
||||
} else {
|
||||
exePath = p
|
||||
u = &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log}
|
||||
// applyStaged is shared by the hourly loop and the control
|
||||
// channel; the once guard keeps a second trigger from
|
||||
// double-waiting on the GPU lock.
|
||||
var once sync.Once
|
||||
applyStaged = func(to string) {
|
||||
if !isService {
|
||||
log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply", "version", to)
|
||||
return
|
||||
}
|
||||
log.Warn("auto-update: staged; restarting once the GPU is idle", "version", to)
|
||||
once.Do(func() {
|
||||
go func() {
|
||||
if waitForIdle(ctx, lk, 24*time.Hour) {
|
||||
log.Warn("auto-update: restarting to apply update")
|
||||
os.Exit(exitCodeUpdate)
|
||||
}
|
||||
}()
|
||||
})
|
||||
restartWhenIdle("update to " + to)
|
||||
}
|
||||
go updateLoop(ctx, cfg.UpdateInterval, log, u, exePath, applyStaged)
|
||||
}
|
||||
}
|
||||
|
||||
// The control channel (status for --monitor, update-now trigger) is
|
||||
// served whenever running as a service, independent of AUTO_UPDATE.
|
||||
// The control channel (status for --monitor, update-now and reload-env
|
||||
// triggers) is served whenever running as a service, independent of
|
||||
// AUTO_UPDATE.
|
||||
if isService {
|
||||
serveControl(ctx, log, u, exePath, applyStaged,
|
||||
statusProvider(cfg, lk, comfySup, health, started))
|
||||
statusProvider(cfg, lk, comfySup, health, started, gw),
|
||||
reloadHandler(cfg, configPath, restartWhenIdle))
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -805,12 +836,34 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
||||
// (idle); health checks skip it instead of logging an outage.
|
||||
var errManagedDown = errors.New("managed upstream intentionally stopped")
|
||||
|
||||
// gpuWatch records the latest VRAM reading from the game detector's poll
|
||||
// loop, for the status channel. Known stays false when game detection is
|
||||
// not configured (no nvidia-smi polling happens then).
|
||||
type gpuWatch struct {
|
||||
mu sync.Mutex
|
||||
usedMB int
|
||||
total int
|
||||
known bool
|
||||
}
|
||||
|
||||
func (g *gpuWatch) set(used, total int) {
|
||||
g.mu.Lock()
|
||||
g.usedMB, g.total, g.known = used, total, true
|
||||
g.mu.Unlock()
|
||||
}
|
||||
|
||||
func (g *gpuWatch) get() (used, total int, known bool) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.usedMB, g.total, g.known
|
||||
}
|
||||
|
||||
// gameLoop polls for foreign GPU holders (a game, another ML job). While one
|
||||
// is detected it holds the lock externally so new LLM and image requests
|
||||
// wait (or are rejected per LLM_BUSY_MODE), and — once in-flight work has
|
||||
// drained — frees VRAM for it: the managed ComfyUI is stopped and Ollama's
|
||||
// resident models are unloaded.
|
||||
func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process) {
|
||||
func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process, gw *gpuWatch) {
|
||||
ticker := time.NewTicker(cfg.GamePollInterval)
|
||||
defer ticker.Stop()
|
||||
held, freed := false, false
|
||||
@@ -824,6 +877,9 @@ func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *gam
|
||||
if err != nil && ctx.Err() == nil {
|
||||
log.Warn("game detection failed", "err", err)
|
||||
}
|
||||
if used, total, verr := game.QueryVRAMMB(ctx); verr == nil {
|
||||
gw.set(used, total)
|
||||
}
|
||||
switch {
|
||||
case len(holders) > 0 && !held:
|
||||
held = true
|
||||
@@ -949,17 +1005,20 @@ func updateLoop(ctx context.Context, interval time.Duration, log *slog.Logger, u
|
||||
|
||||
// serveControl opens the local control channel (named pipe on Windows,
|
||||
// unix socket on Linux) so unprivileged local users can query status
|
||||
// (--monitor) and trigger an update check (--force-update/--update-now)
|
||||
// without admin rights. The update payload is signature-verified regardless
|
||||
// of who asks; triggers are rate-limited to one per minute so the channel
|
||||
// cannot be used to spam restarts. u is nil when AUTO_UPDATE=false.
|
||||
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string), status func() string) {
|
||||
// (--monitor), trigger an update check (--force-update/--update-now) and
|
||||
// poke a config reload (--reload-env) without admin rights. The update
|
||||
// payload is signature-verified regardless of who asks; update triggers
|
||||
// are rate-limited to one per minute so the channel cannot be used to spam
|
||||
// restarts. u is nil when AUTO_UPDATE=false.
|
||||
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string), status func() string, reload func() string) {
|
||||
var mu sync.Mutex
|
||||
var lastTrigger time.Time
|
||||
h := func(cmd string) string {
|
||||
switch cmd {
|
||||
case control.CmdStatus:
|
||||
return "OK " + status()
|
||||
case control.CmdReloadEnv:
|
||||
return reload()
|
||||
case control.CmdUpdateNow:
|
||||
default:
|
||||
return "ERR unknown command: " + cmd
|
||||
@@ -1038,18 +1097,62 @@ type statusSnapshot struct {
|
||||
UptimeS int64 `json:"uptime_s"`
|
||||
Downstreams []statusDownstream `json:"downstreams"`
|
||||
Lock statusLock `json:"lock"`
|
||||
// GPU carries the latest VRAM reading; Known is false when game
|
||||
// detection (and with it nvidia-smi polling) is not configured.
|
||||
GPU statusGPU `json:"gpu"`
|
||||
// MonitorNote is set client-side (never over the wire) when the
|
||||
// monitor's own binary differs from the service's version.
|
||||
MonitorNote string `json:"-"`
|
||||
}
|
||||
|
||||
type statusGPU struct {
|
||||
UsedMB int `json:"used_mb"`
|
||||
TotalMB int `json:"total_mb"`
|
||||
Known bool `json:"known"`
|
||||
}
|
||||
|
||||
// reloadHandler re-reads and validates the service's config file for
|
||||
// CmdReloadEnv. An invalid config is reported and the service keeps running
|
||||
// untouched; a valid, changed config triggers a GPU-idle-gated restart onto
|
||||
// it (same mechanism as staged updates); unchanged is a no-op.
|
||||
func reloadHandler(current config.Config, configPath string, restartWhenIdle func(reason string)) func() string {
|
||||
return func() string {
|
||||
ncfg, err := loadMergedConfig(configPath)
|
||||
if err != nil {
|
||||
return "ERR config invalid: " + err.Error()
|
||||
}
|
||||
changed := diffConfig(current, ncfg)
|
||||
if len(changed) == 0 {
|
||||
return "OK config unchanged"
|
||||
}
|
||||
restartWhenIdle("config reload (" + strings.Join(changed, ", ") + ")")
|
||||
return "OK config valid; restarting once the GPU is idle (changed: " + strings.Join(changed, ", ") + ")"
|
||||
}
|
||||
}
|
||||
|
||||
// diffConfig lists the names of fields whose values differ between two
|
||||
// configs.
|
||||
func diffConfig(a, b config.Config) []string {
|
||||
va, vb := reflect.ValueOf(a), reflect.ValueOf(b)
|
||||
t := va.Type()
|
||||
var out []string
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
if !reflect.DeepEqual(va.Field(i).Interface(), vb.Field(i).Interface()) {
|
||||
out = append(out, t.Field(i).Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// statusProvider assembles the one-line JSON snapshot for CmdStatus.
|
||||
func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time) func() string {
|
||||
func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time, gw *gpuWatch) func() string {
|
||||
return func() string {
|
||||
snap := statusSnapshot{
|
||||
Version: version,
|
||||
UptimeS: int64(time.Since(started).Seconds()),
|
||||
}
|
||||
used, total, known := gw.get()
|
||||
snap.GPU = statusGPU{UsedMB: used, TotalMB: total, Known: known}
|
||||
if cfg.OllamaURL != "" {
|
||||
snap.Downstreams = append(snap.Downstreams, statusDownstream{
|
||||
Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"),
|
||||
|
||||
@@ -21,17 +21,26 @@ const (
|
||||
cCyan = "\x1b[36m"
|
||||
)
|
||||
|
||||
// hotkeysLine is the monitor's footer.
|
||||
const hotkeysLine = " " + cDim + "q quit · u update now" + cReset + "\x1b[K\n"
|
||||
|
||||
// monitorCommand renders a live status view of the running service,
|
||||
// refreshed every second from the control channel. When the service
|
||||
// reports a different version and the executable on disk changed (the
|
||||
// updater replaced it), the monitor restarts itself onto the new binary.
|
||||
// Ctrl+C quits.
|
||||
// Hotkeys: q quits, u triggers an update check on the service.
|
||||
func monitorCommand() int {
|
||||
if !stdoutIsTerminal() {
|
||||
fmt.Fprintln(os.Stderr, "gpu-turnstile: --monitor needs an interactive terminal")
|
||||
return 1
|
||||
}
|
||||
enableVirtualTerminal()
|
||||
restore := enableRawKeys()
|
||||
defer func() {
|
||||
if restore != nil {
|
||||
restore()
|
||||
}
|
||||
}()
|
||||
fmt.Print("\x1b[2J") // clear once; frames then redraw in place
|
||||
defer fmt.Print(cReset + "\n")
|
||||
exe, _ := os.Executable()
|
||||
@@ -39,7 +48,18 @@ func monitorCommand() int {
|
||||
if st, err := os.Stat(exe); err == nil {
|
||||
exeStamp = st.ModTime()
|
||||
}
|
||||
for {
|
||||
|
||||
keys := make(chan byte, 8)
|
||||
go readKeys(keys)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
var note string
|
||||
var noteAt time.Time
|
||||
noteCh := make(chan string, 1)
|
||||
updatePending := false
|
||||
|
||||
poll := func() string {
|
||||
frame := renderWaiting()
|
||||
if reply, err := control.Ask(control.CmdStatus); err == nil {
|
||||
if msg, ok := strings.CutPrefix(reply, "OK "); ok {
|
||||
@@ -50,16 +70,75 @@ func monitorCommand() int {
|
||||
fmt.Print("\x1b[2J\x1b[H")
|
||||
fmt.Printf("gpu-turnstile: service updated to %s — restarting the monitor\n", snap.Version)
|
||||
restartSelf(exe, "--monitor")
|
||||
return 0
|
||||
return "" // re-execed; this process exits below
|
||||
}
|
||||
snap.MonitorNote = fmt.Sprintf("note: the service runs %s, this monitor is %s", snap.Version, version)
|
||||
}
|
||||
if note != "" {
|
||||
snap.MonitorNote = note
|
||||
}
|
||||
frame = renderMonitor(snap, termWidth())
|
||||
}
|
||||
}
|
||||
}
|
||||
return frame
|
||||
}
|
||||
|
||||
for {
|
||||
frame := poll()
|
||||
if frame == "" {
|
||||
return 0 // restartSelf fired
|
||||
}
|
||||
fmt.Print("\x1b[H" + frame + "\x1b[J") // home, frame, clear below
|
||||
time.Sleep(time.Second)
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if note != "" && time.Since(noteAt) > 15*time.Second {
|
||||
note = ""
|
||||
}
|
||||
case k, ok := <-keys:
|
||||
if !ok {
|
||||
keys = nil
|
||||
continue
|
||||
}
|
||||
switch k {
|
||||
case 'q', 'Q', 3: // q or Ctrl+C (raw mode delivers it as a byte)
|
||||
return 0
|
||||
case 'u', 'U':
|
||||
if !updatePending {
|
||||
updatePending = true
|
||||
note, noteAt = "checking for updates…", time.Now()
|
||||
go func() {
|
||||
reply, err := control.Ask(control.CmdUpdateNow)
|
||||
if err != nil {
|
||||
noteCh <- "update: no answer from the service"
|
||||
return
|
||||
}
|
||||
msg := strings.TrimPrefix(reply, "OK ")
|
||||
msg = strings.TrimPrefix(msg, "ERR ")
|
||||
noteCh <- "update: " + msg
|
||||
}()
|
||||
}
|
||||
}
|
||||
case n := <-noteCh:
|
||||
updatePending = false
|
||||
note, noteAt = n, time.Now()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readKeys reads single keypresses from stdin (raw mode was enabled by the
|
||||
// caller) and delivers them until stdin fails.
|
||||
func readKeys(keys chan<- byte) {
|
||||
defer close(keys)
|
||||
buf := make([]byte, 1)
|
||||
for {
|
||||
n, err := os.Stdin.Read(buf)
|
||||
if n > 0 {
|
||||
keys <- buf[0]
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,7 +161,7 @@ func restartSelf(exe string, args ...string) {
|
||||
}
|
||||
|
||||
func renderWaiting() string {
|
||||
return cDim + " gpu-turnstile — waiting for a running service…" + cReset + "\x1b[K\n"
|
||||
return cDim + " gpu-turnstile — waiting for a running service…" + cReset + "\x1b[K\n\x1b[K\n" + hotkeysLine
|
||||
}
|
||||
|
||||
// renderMonitor draws one full frame. Each line ends with \x1b[K (clear to
|
||||
@@ -117,12 +196,31 @@ func renderMonitor(snap statusSnapshot, width int) string {
|
||||
b.WriteString(fmt.Sprintf(" Queue: %s%d image job(s) waiting%s\x1b[K\n",
|
||||
cYellow, snap.Lock.ImageQueue, cReset))
|
||||
}
|
||||
if snap.GPU.Known {
|
||||
b.WriteString(" GPU: " + renderVRAM(snap.GPU.UsedMB, snap.GPU.TotalMB) + "\x1b[K\n")
|
||||
}
|
||||
if snap.MonitorNote != "" {
|
||||
b.WriteString(" " + cYellow + snap.MonitorNote + cReset + "\x1b[K\n")
|
||||
}
|
||||
b.WriteString("\x1b[K\n")
|
||||
b.WriteString(hotkeysLine)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderVRAM renders "4.2 / 16.0 GiB used" (or MiB below 1 GiB).
|
||||
func renderVRAM(used, total int) string {
|
||||
format := func(mb int) string {
|
||||
if mb >= 1024 {
|
||||
return fmt.Sprintf("%.1f GiB", float64(mb)/1024)
|
||||
}
|
||||
return fmt.Sprintf("%d MiB", mb)
|
||||
}
|
||||
if total > 0 {
|
||||
return format(used) + " / " + format(total) + " used"
|
||||
}
|
||||
return format(used) + " used"
|
||||
}
|
||||
|
||||
// printableLen counts characters without ANSI escapes (ASCII-only content).
|
||||
func printableLen(s string) int { return len(s) }
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"gpu-turnstile/internal/config"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -39,3 +40,17 @@ func TestFmtDur(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffConfig(t *testing.T) {
|
||||
a := config.Defaults()
|
||||
b := a
|
||||
if got := diffConfig(a, b); len(got) != 0 {
|
||||
t.Fatalf("identical configs: got %v", got)
|
||||
}
|
||||
b.LogLevel = -4
|
||||
b.GameProcs = []string{"game.exe"}
|
||||
got := diffConfig(a, b)
|
||||
if len(got) != 2 || got[0] != "GameProcs" || got[1] != "LogLevel" {
|
||||
t.Fatalf("got %v, want [GameProcs LogLevel]", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,3 +19,21 @@ func termWidth() int {
|
||||
}
|
||||
return int(ws.Col)
|
||||
}
|
||||
|
||||
// enableRawKeys switches the terminal to per-keypress mode (ICANON and ECHO
|
||||
// off) and returns the restore function, nil when stdin is not a terminal.
|
||||
func enableRawKeys() func() {
|
||||
fd := int(os.Stdin.Fd())
|
||||
term, err := unix.IoctlGetTermios(fd, unix.TCGETS)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
raw := *term
|
||||
raw.Lflag &^= unix.ICANON | unix.ECHO
|
||||
raw.Cc[unix.VMIN] = 1
|
||||
raw.Cc[unix.VTIME] = 0
|
||||
if err := unix.IoctlSetTermios(fd, unix.TCSETS, &raw); err != nil {
|
||||
return nil
|
||||
}
|
||||
return func() { unix.IoctlSetTermios(fd, unix.TCSETS, term) } //nolint:errcheck
|
||||
}
|
||||
|
||||
@@ -27,3 +27,23 @@ func termWidth() int {
|
||||
}
|
||||
return int(info.Window.Right-info.Window.Left) + 1
|
||||
}
|
||||
|
||||
// enableRawKeys puts the console's stdin into per-keypress mode (no line
|
||||
// buffering, no echo) and returns the restore function. When stdin is not a
|
||||
// real console (mintty/Git Bash pipes) it returns nil: ptys already deliver
|
||||
// keystrokes immediately.
|
||||
func enableRawKeys() func() {
|
||||
h := windows.Handle(os.Stdin.Fd())
|
||||
var mode uint32
|
||||
if err := windows.GetConsoleMode(h, &mode); err != nil {
|
||||
return nil
|
||||
}
|
||||
const (
|
||||
enableLineInput = 0x0002
|
||||
enableEchoInput = 0x0004
|
||||
)
|
||||
if err := windows.SetConsoleMode(h, mode&^(enableLineInput|enableEchoInput)); err != nil {
|
||||
return nil
|
||||
}
|
||||
return func() { windows.SetConsoleMode(h, mode) } //nolint:errcheck
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
# ComfyUI --listen 0.0.0.0 --port 8189).
|
||||
services:
|
||||
gpu-turnstile:
|
||||
image: git.rambossek.at/public/gpu-turnstile:v0.2.7
|
||||
image: git.rambossek.at/public/gpu-turnstile:v0.2.8
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Each consumer is enabled by setting its URL; leave one unset to
|
||||
|
||||
@@ -26,6 +26,10 @@ const CmdUpdateNow = "update-now"
|
||||
// CmdStatus asks for a one-line JSON status snapshot (monitor mode).
|
||||
const CmdStatus = "status"
|
||||
|
||||
// CmdReloadEnv asks the service to re-read and validate its config file,
|
||||
// and to restart onto it (once the GPU is idle) when it changed.
|
||||
const CmdReloadEnv = "reload-env"
|
||||
|
||||
// ErrUnavailable means no running service offers the control channel.
|
||||
var ErrUnavailable = errors.New("control channel unavailable")
|
||||
|
||||
|
||||
@@ -132,6 +132,29 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
|
||||
return parseComputeApps(string(out))
|
||||
}
|
||||
|
||||
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
|
||||
// Unlike the per-process list this works under WDDM too.
|
||||
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
|
||||
out, err := exec.CommandContext(ctx, "nvidia-smi",
|
||||
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
|
||||
}
|
||||
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
|
||||
}
|
||||
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
|
||||
}
|
||||
return used, total, nil
|
||||
}
|
||||
|
||||
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
|
||||
// units). Unsupported rows ("N/A" on WDDM) are skipped.
|
||||
func parseComputeApps(out string) ([]computeApp, error) {
|
||||
|
||||
Reference in New Issue
Block a user