Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4bd5f34ce7 | ||
|
|
b9d3f91403 | ||
|
|
98d2d714ca |
+136
-33
@@ -16,6 +16,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -39,7 +40,8 @@ import (
|
|||||||
var version = "dev"
|
var version = "dev"
|
||||||
|
|
||||||
// exitCodeUpdate tells the service recovery configuration to restart the
|
// exitCodeUpdate tells the service recovery configuration to restart the
|
||||||
// process: a signed update has been staged and the GPU lock is idle.
|
// process: a signed update has been staged or a changed config reload was
|
||||||
|
// requested, and the GPU lock is idle.
|
||||||
const exitCodeUpdate = 3
|
const exitCodeUpdate = 3
|
||||||
|
|
||||||
// exitCodeStaged is returned by an elevated --force-update child when it
|
// exitCodeStaged is returned by an elevated --force-update child when it
|
||||||
@@ -59,7 +61,7 @@ func stdoutIsTerminal() bool {
|
|||||||
// --install-service / --remove-service switches, --no-copy, -h/--help,
|
// --install-service / --remove-service switches, --no-copy, -h/--help,
|
||||||
// -v/--version, --force-update and the hidden --elevated-child marker from
|
// -v/--version, --force-update and the hidden --elevated-child marker from
|
||||||
// args.
|
// args.
|
||||||
func parseFlags(args []string) (configPath string, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild bool, rest []string) {
|
func parseFlags(args []string) (configPath string, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild bool, rest []string) {
|
||||||
rest = args[:0]
|
rest = args[:0]
|
||||||
for i := 0; i < len(args); i++ {
|
for i := 0; i < len(args); i++ {
|
||||||
switch {
|
switch {
|
||||||
@@ -82,6 +84,8 @@ func parseFlags(args []string) (configPath string, install, remove, noCopy, help
|
|||||||
forceUpdate = true
|
forceUpdate = true
|
||||||
case args[i] == "--update-now" || args[i] == "-update-now":
|
case args[i] == "--update-now" || args[i] == "-update-now":
|
||||||
updateNow = true
|
updateNow = true
|
||||||
|
case args[i] == "--reload-env" || args[i] == "-reload-env":
|
||||||
|
reloadEnv = true
|
||||||
case args[i] == "--monitor" || args[i] == "-monitor" || args[i] == "-m":
|
case args[i] == "--monitor" || args[i] == "-monitor" || args[i] == "-m":
|
||||||
monitor = true
|
monitor = true
|
||||||
case args[i] == "--elevated-child":
|
case args[i] == "--elevated-child":
|
||||||
@@ -90,7 +94,7 @@ func parseFlags(args []string) (configPath string, install, remove, noCopy, help
|
|||||||
rest = append(rest, args[i])
|
rest = append(rest, args[i])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild, rest
|
return configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild, rest
|
||||||
}
|
}
|
||||||
|
|
||||||
// versionLine is printed at the top of every help and error screen.
|
// versionLine is printed at the top of every help and error screen.
|
||||||
@@ -109,6 +113,9 @@ Usage:
|
|||||||
(no admin needed when the service runs)
|
(no admin needed when the service runs)
|
||||||
gpu-turnstile --update-now like --force-update, but only
|
gpu-turnstile --update-now like --force-update, but only
|
||||||
through the running service
|
through the running service
|
||||||
|
gpu-turnstile --reload-env make the service re-read and
|
||||||
|
validate its config file, then
|
||||||
|
restart onto it if it changed
|
||||||
gpu-turnstile -m | --monitor live status view (downstreams,
|
gpu-turnstile -m | --monitor live status view (downstreams,
|
||||||
GPU lock, queue); Ctrl+C quits
|
GPU lock, queue); Ctrl+C quits
|
||||||
gpu-turnstile -h | --help this help
|
gpu-turnstile -h | --help this help
|
||||||
@@ -141,12 +148,12 @@ func fatalUsage(format string, args ...any) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, monitor, elevatedChild, args := parseFlags(os.Args[1:])
|
configPath, install, remove, noCopy, help, showVersion, forceUpdate, updateNow, reloadEnv, monitor, elevatedChild, args := parseFlags(os.Args[1:])
|
||||||
if showVersion {
|
if showVersion {
|
||||||
fmt.Println(version)
|
fmt.Println(version)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
bare := configPath == "" && !install && !remove && !forceUpdate && !updateNow && !monitor && !elevatedChild && len(args) == 0
|
bare := configPath == "" && !install && !remove && !forceUpdate && !updateNow && !reloadEnv && !monitor && !elevatedChild && len(args) == 0
|
||||||
if help || (bare && stdoutIsTerminal()) {
|
if help || (bare && stdoutIsTerminal()) {
|
||||||
// Bare invocation in a terminal (e.g. double-clicked on Windows)
|
// Bare invocation in a terminal (e.g. double-clicked on Windows)
|
||||||
// shows the help instead of starting a proxy window with no visible
|
// shows the help instead of starting a proxy window with no visible
|
||||||
@@ -171,7 +178,9 @@ func main() {
|
|||||||
fatalUsage("error: --force-update cannot be combined with --install-service/--remove-service")
|
fatalUsage("error: --force-update cannot be combined with --install-service/--remove-service")
|
||||||
case updateNow && (install || remove || forceUpdate):
|
case updateNow && (install || remove || forceUpdate):
|
||||||
fatalUsage("error: --update-now cannot be combined with other commands")
|
fatalUsage("error: --update-now cannot be combined with other commands")
|
||||||
case monitor && (install || remove || forceUpdate || updateNow):
|
case reloadEnv && (install || remove || forceUpdate || updateNow):
|
||||||
|
fatalUsage("error: --reload-env cannot be combined with other commands")
|
||||||
|
case monitor && (install || remove || forceUpdate || updateNow || reloadEnv):
|
||||||
fatalUsage("error: --monitor cannot be combined with other commands")
|
fatalUsage("error: --monitor cannot be combined with other commands")
|
||||||
case install:
|
case install:
|
||||||
os.Exit(serviceCommand(configPath, true, noCopy, elevatedChild))
|
os.Exit(serviceCommand(configPath, true, noCopy, elevatedChild))
|
||||||
@@ -181,6 +190,8 @@ func main() {
|
|||||||
os.Exit(forceUpdateCommand(configPath, elevatedChild))
|
os.Exit(forceUpdateCommand(configPath, elevatedChild))
|
||||||
case updateNow:
|
case updateNow:
|
||||||
os.Exit(updateNowCommand())
|
os.Exit(updateNowCommand())
|
||||||
|
case reloadEnv:
|
||||||
|
os.Exit(reloadEnvCommand())
|
||||||
case monitor:
|
case monitor:
|
||||||
os.Exit(monitorCommand())
|
os.Exit(monitorCommand())
|
||||||
}
|
}
|
||||||
@@ -202,7 +213,7 @@ func main() {
|
|||||||
syncEnvFile(resolveConfigPath(configPath), cfg.LogFile, log)
|
syncEnvFile(resolveConfigPath(configPath), cfg.LogFile, log)
|
||||||
|
|
||||||
if service.IsService() {
|
if service.IsService() {
|
||||||
if err := service.Run(func(ctx context.Context) error { return run(ctx, cfg, log, logOut, true) }); err != nil {
|
if err := service.Run(func(ctx context.Context) error { return run(ctx, cfg, log, logOut, true, configPath) }); err != nil {
|
||||||
log.Error("service failed", "err", err)
|
log.Error("service failed", "err", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
@@ -210,7 +221,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
if err := run(ctx, cfg, log, logOut, false); err != nil {
|
if err := run(ctx, cfg, log, logOut, false, configPath); err != nil {
|
||||||
log.Error("listener failed", "err", err)
|
log.Error("listener failed", "err", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
@@ -451,6 +462,18 @@ func updateNowCommand() int {
|
|||||||
return printControlReply(reply)
|
return printControlReply(reply)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reloadEnvCommand asks the running service, over the control channel, to
|
||||||
|
// re-read and validate its config file. The client sends no path — the
|
||||||
|
// service only ever re-reads its own configured file.
|
||||||
|
func reloadEnvCommand() int {
|
||||||
|
reply, err := control.Ask(control.CmdReloadEnv)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s\n\ngpu-turnstile: no running service to ask\n", versionLine())
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return printControlReply(reply)
|
||||||
|
}
|
||||||
|
|
||||||
// reportElevatedUpdate prints the parent's summary of an elevated
|
// reportElevatedUpdate prints the parent's summary of an elevated
|
||||||
// --force-update child: exitCodeStaged means the child staged a new binary,
|
// --force-update child: exitCodeStaged means the child staged a new binary,
|
||||||
// 0 means it found nothing to do. to is the tag the parent's own check
|
// 0 means it found nothing to do. to is the tag the parent's own check
|
||||||
@@ -531,7 +554,7 @@ func managedComfyCommand(cfg config.Config) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Writer, isService bool) error {
|
func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Writer, isService bool, configPath string) error {
|
||||||
// ComfyUI can run as a managed child — COMFY_CMD verbatim, or the
|
// ComfyUI can run as a managed child — COMFY_CMD verbatim, or the
|
||||||
// standard venv layout derived from COMFY_DIR alone: started on demand
|
// standard venv layout derived from COMFY_DIR alone: started on demand
|
||||||
// by the proxy, stopped after COMFY_IDLE_TIMEOUT idle (and on shutdown)
|
// by the proxy, stopped after COMFY_IDLE_TIMEOUT idle (and on shutdown)
|
||||||
@@ -705,9 +728,11 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
|||||||
|
|
||||||
// Foreign GPU holders (games, other ML jobs) — enabled by GAME_PROCS
|
// Foreign GPU holders (games, other ML jobs) — enabled by GAME_PROCS
|
||||||
// and/or GPU_FOREIGN_VRAM_MB — hold the lock externally while they run.
|
// and/or GPU_FOREIGN_VRAM_MB — hold the lock externally while they run.
|
||||||
|
// gw collects the VRAM reading for the status channel.
|
||||||
|
gw := &gpuWatch{}
|
||||||
if len(cfg.GameProcs) > 0 || cfg.GPUForeignVRAMMB > 0 {
|
if len(cfg.GameProcs) > 0 || cfg.GPUForeignVRAMMB > 0 {
|
||||||
det := game.New(cfg.GameProcs, cfg.GPUForeignVRAMMB, cfg.GPUIgnoreProcs, log)
|
det := game.New(cfg.GameProcs, cfg.GPUForeignVRAMMB, cfg.GPUIgnoreProcs, log)
|
||||||
go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup)
|
go gameLoop(ctx, cfg, log, det, lk, ollamaClient, comfySup, gw)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bind the listeners up front so a port conflict fails fast and the
|
// Bind the listeners up front so a port conflict fails fast and the
|
||||||
@@ -747,6 +772,22 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
|||||||
|
|
||||||
var u *update.Updater
|
var u *update.Updater
|
||||||
var exePath string
|
var exePath string
|
||||||
|
// restartWhenIdle exits with exitCodeUpdate once the GPU lock is idle;
|
||||||
|
// the service recovery configuration brings the process back. Shared by
|
||||||
|
// staged updates and config reloads; the once guard makes repeated
|
||||||
|
// triggers idempotent.
|
||||||
|
var restartOnce sync.Once
|
||||||
|
restartWhenIdle := func(reason string) {
|
||||||
|
log.Warn("restarting once the GPU is idle", "reason", reason)
|
||||||
|
restartOnce.Do(func() {
|
||||||
|
go func() {
|
||||||
|
if waitForIdle(ctx, lk, 24*time.Hour) {
|
||||||
|
log.Warn("restarting now", "reason", reason)
|
||||||
|
os.Exit(exitCodeUpdate)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
})
|
||||||
|
}
|
||||||
applyStaged := func(to string) {}
|
applyStaged := func(to string) {}
|
||||||
if cfg.AutoUpdate {
|
if cfg.AutoUpdate {
|
||||||
if p, err := os.Executable(); err != nil {
|
if p, err := os.Executable(); err != nil {
|
||||||
@@ -754,34 +795,24 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
|||||||
} else {
|
} else {
|
||||||
exePath = p
|
exePath = p
|
||||||
u = &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log}
|
u = &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log}
|
||||||
// applyStaged is shared by the hourly loop and the control
|
|
||||||
// channel; the once guard keeps a second trigger from
|
|
||||||
// double-waiting on the GPU lock.
|
|
||||||
var once sync.Once
|
|
||||||
applyStaged = func(to string) {
|
applyStaged = func(to string) {
|
||||||
if !isService {
|
if !isService {
|
||||||
log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply", "version", to)
|
log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply", "version", to)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
log.Warn("auto-update: staged; restarting once the GPU is idle", "version", to)
|
restartWhenIdle("update to " + to)
|
||||||
once.Do(func() {
|
|
||||||
go func() {
|
|
||||||
if waitForIdle(ctx, lk, 24*time.Hour) {
|
|
||||||
log.Warn("auto-update: restarting to apply update")
|
|
||||||
os.Exit(exitCodeUpdate)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
go updateLoop(ctx, cfg.UpdateInterval, log, u, exePath, applyStaged)
|
go updateLoop(ctx, cfg.UpdateInterval, log, u, exePath, applyStaged)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The control channel (status for --monitor, update-now trigger) is
|
// The control channel (status for --monitor, update-now and reload-env
|
||||||
// served whenever running as a service, independent of AUTO_UPDATE.
|
// triggers) is served whenever running as a service, independent of
|
||||||
|
// AUTO_UPDATE.
|
||||||
if isService {
|
if isService {
|
||||||
serveControl(ctx, log, u, exePath, applyStaged,
|
serveControl(ctx, log, u, exePath, applyStaged,
|
||||||
statusProvider(cfg, lk, comfySup, health, started))
|
statusProvider(cfg, lk, comfySup, health, started, gw),
|
||||||
|
reloadHandler(cfg, configPath, restartWhenIdle))
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -805,12 +836,34 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
|||||||
// (idle); health checks skip it instead of logging an outage.
|
// (idle); health checks skip it instead of logging an outage.
|
||||||
var errManagedDown = errors.New("managed upstream intentionally stopped")
|
var errManagedDown = errors.New("managed upstream intentionally stopped")
|
||||||
|
|
||||||
|
// gpuWatch records the latest VRAM reading from the game detector's poll
|
||||||
|
// loop, for the status channel. Known stays false when game detection is
|
||||||
|
// not configured (no nvidia-smi polling happens then).
|
||||||
|
type gpuWatch struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
usedMB int
|
||||||
|
total int
|
||||||
|
known bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gpuWatch) set(used, total int) {
|
||||||
|
g.mu.Lock()
|
||||||
|
g.usedMB, g.total, g.known = used, total, true
|
||||||
|
g.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gpuWatch) get() (used, total int, known bool) {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
return g.usedMB, g.total, g.known
|
||||||
|
}
|
||||||
|
|
||||||
// gameLoop polls for foreign GPU holders (a game, another ML job). While one
|
// gameLoop polls for foreign GPU holders (a game, another ML job). While one
|
||||||
// is detected it holds the lock externally so new LLM and image requests
|
// is detected it holds the lock externally so new LLM and image requests
|
||||||
// wait (or are rejected per LLM_BUSY_MODE), and — once in-flight work has
|
// wait (or are rejected per LLM_BUSY_MODE), and — once in-flight work has
|
||||||
// drained — frees VRAM for it: the managed ComfyUI is stopped and Ollama's
|
// drained — frees VRAM for it: the managed ComfyUI is stopped and Ollama's
|
||||||
// resident models are unloaded.
|
// resident models are unloaded.
|
||||||
func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process) {
|
func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *game.Detector, lk *lock.Lock, ollamaClient *ollama.Client, comfySup *supervise.Process, gw *gpuWatch) {
|
||||||
ticker := time.NewTicker(cfg.GamePollInterval)
|
ticker := time.NewTicker(cfg.GamePollInterval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
held, freed := false, false
|
held, freed := false, false
|
||||||
@@ -824,6 +877,9 @@ func gameLoop(ctx context.Context, cfg config.Config, log *slog.Logger, det *gam
|
|||||||
if err != nil && ctx.Err() == nil {
|
if err != nil && ctx.Err() == nil {
|
||||||
log.Warn("game detection failed", "err", err)
|
log.Warn("game detection failed", "err", err)
|
||||||
}
|
}
|
||||||
|
if used, total, verr := game.QueryVRAMMB(ctx); verr == nil {
|
||||||
|
gw.set(used, total)
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case len(holders) > 0 && !held:
|
case len(holders) > 0 && !held:
|
||||||
held = true
|
held = true
|
||||||
@@ -949,17 +1005,20 @@ func updateLoop(ctx context.Context, interval time.Duration, log *slog.Logger, u
|
|||||||
|
|
||||||
// serveControl opens the local control channel (named pipe on Windows,
|
// serveControl opens the local control channel (named pipe on Windows,
|
||||||
// unix socket on Linux) so unprivileged local users can query status
|
// unix socket on Linux) so unprivileged local users can query status
|
||||||
// (--monitor) and trigger an update check (--force-update/--update-now)
|
// (--monitor), trigger an update check (--force-update/--update-now) and
|
||||||
// without admin rights. The update payload is signature-verified regardless
|
// poke a config reload (--reload-env) without admin rights. The update
|
||||||
// of who asks; triggers are rate-limited to one per minute so the channel
|
// payload is signature-verified regardless of who asks; update triggers
|
||||||
// cannot be used to spam restarts. u is nil when AUTO_UPDATE=false.
|
// are rate-limited to one per minute so the channel cannot be used to spam
|
||||||
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string), status func() string) {
|
// restarts. u is nil when AUTO_UPDATE=false.
|
||||||
|
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string), status func() string, reload func() string) {
|
||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
var lastTrigger time.Time
|
var lastTrigger time.Time
|
||||||
h := func(cmd string) string {
|
h := func(cmd string) string {
|
||||||
switch cmd {
|
switch cmd {
|
||||||
case control.CmdStatus:
|
case control.CmdStatus:
|
||||||
return "OK " + status()
|
return "OK " + status()
|
||||||
|
case control.CmdReloadEnv:
|
||||||
|
return reload()
|
||||||
case control.CmdUpdateNow:
|
case control.CmdUpdateNow:
|
||||||
default:
|
default:
|
||||||
return "ERR unknown command: " + cmd
|
return "ERR unknown command: " + cmd
|
||||||
@@ -1038,18 +1097,62 @@ type statusSnapshot struct {
|
|||||||
UptimeS int64 `json:"uptime_s"`
|
UptimeS int64 `json:"uptime_s"`
|
||||||
Downstreams []statusDownstream `json:"downstreams"`
|
Downstreams []statusDownstream `json:"downstreams"`
|
||||||
Lock statusLock `json:"lock"`
|
Lock statusLock `json:"lock"`
|
||||||
|
// GPU carries the latest VRAM reading; Known is false when game
|
||||||
|
// detection (and with it nvidia-smi polling) is not configured.
|
||||||
|
GPU statusGPU `json:"gpu"`
|
||||||
// MonitorNote is set client-side (never over the wire) when the
|
// MonitorNote is set client-side (never over the wire) when the
|
||||||
// monitor's own binary differs from the service's version.
|
// monitor's own binary differs from the service's version.
|
||||||
MonitorNote string `json:"-"`
|
MonitorNote string `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type statusGPU struct {
|
||||||
|
UsedMB int `json:"used_mb"`
|
||||||
|
TotalMB int `json:"total_mb"`
|
||||||
|
Known bool `json:"known"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// reloadHandler re-reads and validates the service's config file for
|
||||||
|
// CmdReloadEnv. An invalid config is reported and the service keeps running
|
||||||
|
// untouched; a valid, changed config triggers a GPU-idle-gated restart onto
|
||||||
|
// it (same mechanism as staged updates); unchanged is a no-op.
|
||||||
|
func reloadHandler(current config.Config, configPath string, restartWhenIdle func(reason string)) func() string {
|
||||||
|
return func() string {
|
||||||
|
ncfg, err := loadMergedConfig(configPath)
|
||||||
|
if err != nil {
|
||||||
|
return "ERR config invalid: " + err.Error()
|
||||||
|
}
|
||||||
|
changed := diffConfig(current, ncfg)
|
||||||
|
if len(changed) == 0 {
|
||||||
|
return "OK config unchanged"
|
||||||
|
}
|
||||||
|
restartWhenIdle("config reload (" + strings.Join(changed, ", ") + ")")
|
||||||
|
return "OK config valid; restarting once the GPU is idle (changed: " + strings.Join(changed, ", ") + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// diffConfig lists the names of fields whose values differ between two
|
||||||
|
// configs.
|
||||||
|
func diffConfig(a, b config.Config) []string {
|
||||||
|
va, vb := reflect.ValueOf(a), reflect.ValueOf(b)
|
||||||
|
t := va.Type()
|
||||||
|
var out []string
|
||||||
|
for i := 0; i < t.NumField(); i++ {
|
||||||
|
if !reflect.DeepEqual(va.Field(i).Interface(), vb.Field(i).Interface()) {
|
||||||
|
out = append(out, t.Field(i).Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// statusProvider assembles the one-line JSON snapshot for CmdStatus.
|
// statusProvider assembles the one-line JSON snapshot for CmdStatus.
|
||||||
func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time) func() string {
|
func statusProvider(cfg config.Config, lk *lock.Lock, comfySup *supervise.Process, health *healthTracker, started time.Time, gw *gpuWatch) func() string {
|
||||||
return func() string {
|
return func() string {
|
||||||
snap := statusSnapshot{
|
snap := statusSnapshot{
|
||||||
Version: version,
|
Version: version,
|
||||||
UptimeS: int64(time.Since(started).Seconds()),
|
UptimeS: int64(time.Since(started).Seconds()),
|
||||||
}
|
}
|
||||||
|
used, total, known := gw.get()
|
||||||
|
snap.GPU = statusGPU{UsedMB: used, TotalMB: total, Known: known}
|
||||||
if cfg.OllamaURL != "" {
|
if cfg.OllamaURL != "" {
|
||||||
snap.Downstreams = append(snap.Downstreams, statusDownstream{
|
snap.Downstreams = append(snap.Downstreams, statusDownstream{
|
||||||
Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"),
|
Name: "ollama", URL: cfg.OllamaURL, Up: health.get("ollama"),
|
||||||
|
|||||||
@@ -21,17 +21,26 @@ const (
|
|||||||
cCyan = "\x1b[36m"
|
cCyan = "\x1b[36m"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// hotkeysLine is the monitor's footer.
|
||||||
|
const hotkeysLine = " " + cDim + "q quit · u update now" + cReset + "\x1b[K\n"
|
||||||
|
|
||||||
// monitorCommand renders a live status view of the running service,
|
// monitorCommand renders a live status view of the running service,
|
||||||
// refreshed every second from the control channel. When the service
|
// refreshed every second from the control channel. When the service
|
||||||
// reports a different version and the executable on disk changed (the
|
// reports a different version and the executable on disk changed (the
|
||||||
// updater replaced it), the monitor restarts itself onto the new binary.
|
// updater replaced it), the monitor restarts itself onto the new binary.
|
||||||
// Ctrl+C quits.
|
// Hotkeys: q quits, u triggers an update check on the service.
|
||||||
func monitorCommand() int {
|
func monitorCommand() int {
|
||||||
if !stdoutIsTerminal() {
|
if !stdoutIsTerminal() {
|
||||||
fmt.Fprintln(os.Stderr, "gpu-turnstile: --monitor needs an interactive terminal")
|
fmt.Fprintln(os.Stderr, "gpu-turnstile: --monitor needs an interactive terminal")
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
enableVirtualTerminal()
|
enableVirtualTerminal()
|
||||||
|
restore := enableRawKeys()
|
||||||
|
defer func() {
|
||||||
|
if restore != nil {
|
||||||
|
restore()
|
||||||
|
}
|
||||||
|
}()
|
||||||
fmt.Print("\x1b[2J") // clear once; frames then redraw in place
|
fmt.Print("\x1b[2J") // clear once; frames then redraw in place
|
||||||
defer fmt.Print(cReset + "\n")
|
defer fmt.Print(cReset + "\n")
|
||||||
exe, _ := os.Executable()
|
exe, _ := os.Executable()
|
||||||
@@ -39,7 +48,18 @@ func monitorCommand() int {
|
|||||||
if st, err := os.Stat(exe); err == nil {
|
if st, err := os.Stat(exe); err == nil {
|
||||||
exeStamp = st.ModTime()
|
exeStamp = st.ModTime()
|
||||||
}
|
}
|
||||||
for {
|
|
||||||
|
keys := make(chan byte, 8)
|
||||||
|
go readKeys(keys)
|
||||||
|
ticker := time.NewTicker(time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
var note string
|
||||||
|
var noteAt time.Time
|
||||||
|
noteCh := make(chan string, 1)
|
||||||
|
updatePending := false
|
||||||
|
|
||||||
|
poll := func() string {
|
||||||
frame := renderWaiting()
|
frame := renderWaiting()
|
||||||
if reply, err := control.Ask(control.CmdStatus); err == nil {
|
if reply, err := control.Ask(control.CmdStatus); err == nil {
|
||||||
if msg, ok := strings.CutPrefix(reply, "OK "); ok {
|
if msg, ok := strings.CutPrefix(reply, "OK "); ok {
|
||||||
@@ -50,16 +70,75 @@ func monitorCommand() int {
|
|||||||
fmt.Print("\x1b[2J\x1b[H")
|
fmt.Print("\x1b[2J\x1b[H")
|
||||||
fmt.Printf("gpu-turnstile: service updated to %s — restarting the monitor\n", snap.Version)
|
fmt.Printf("gpu-turnstile: service updated to %s — restarting the monitor\n", snap.Version)
|
||||||
restartSelf(exe, "--monitor")
|
restartSelf(exe, "--monitor")
|
||||||
return 0
|
return "" // re-execed; this process exits below
|
||||||
}
|
}
|
||||||
snap.MonitorNote = fmt.Sprintf("note: the service runs %s, this monitor is %s", snap.Version, version)
|
snap.MonitorNote = fmt.Sprintf("note: the service runs %s, this monitor is %s", snap.Version, version)
|
||||||
}
|
}
|
||||||
|
if note != "" {
|
||||||
|
snap.MonitorNote = note
|
||||||
|
}
|
||||||
frame = renderMonitor(snap, termWidth())
|
frame = renderMonitor(snap, termWidth())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return frame
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
frame := poll()
|
||||||
|
if frame == "" {
|
||||||
|
return 0 // restartSelf fired
|
||||||
|
}
|
||||||
fmt.Print("\x1b[H" + frame + "\x1b[J") // home, frame, clear below
|
fmt.Print("\x1b[H" + frame + "\x1b[J") // home, frame, clear below
|
||||||
time.Sleep(time.Second)
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
if note != "" && time.Since(noteAt) > 15*time.Second {
|
||||||
|
note = ""
|
||||||
|
}
|
||||||
|
case k, ok := <-keys:
|
||||||
|
if !ok {
|
||||||
|
keys = nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch k {
|
||||||
|
case 'q', 'Q', 3: // q or Ctrl+C (raw mode delivers it as a byte)
|
||||||
|
return 0
|
||||||
|
case 'u', 'U':
|
||||||
|
if !updatePending {
|
||||||
|
updatePending = true
|
||||||
|
note, noteAt = "checking for updates…", time.Now()
|
||||||
|
go func() {
|
||||||
|
reply, err := control.Ask(control.CmdUpdateNow)
|
||||||
|
if err != nil {
|
||||||
|
noteCh <- "update: no answer from the service"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
msg := strings.TrimPrefix(reply, "OK ")
|
||||||
|
msg = strings.TrimPrefix(msg, "ERR ")
|
||||||
|
noteCh <- "update: " + msg
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case n := <-noteCh:
|
||||||
|
updatePending = false
|
||||||
|
note, noteAt = n, time.Now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readKeys reads single keypresses from stdin (raw mode was enabled by the
|
||||||
|
// caller) and delivers them until stdin fails.
|
||||||
|
func readKeys(keys chan<- byte) {
|
||||||
|
defer close(keys)
|
||||||
|
buf := make([]byte, 1)
|
||||||
|
for {
|
||||||
|
n, err := os.Stdin.Read(buf)
|
||||||
|
if n > 0 {
|
||||||
|
keys <- buf[0]
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -82,7 +161,7 @@ func restartSelf(exe string, args ...string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func renderWaiting() string {
|
func renderWaiting() string {
|
||||||
return cDim + " gpu-turnstile — waiting for a running service…" + cReset + "\x1b[K\n"
|
return cDim + " gpu-turnstile — waiting for a running service…" + cReset + "\x1b[K\n\x1b[K\n" + hotkeysLine
|
||||||
}
|
}
|
||||||
|
|
||||||
// renderMonitor draws one full frame. Each line ends with \x1b[K (clear to
|
// renderMonitor draws one full frame. Each line ends with \x1b[K (clear to
|
||||||
@@ -117,12 +196,31 @@ func renderMonitor(snap statusSnapshot, width int) string {
|
|||||||
b.WriteString(fmt.Sprintf(" Queue: %s%d image job(s) waiting%s\x1b[K\n",
|
b.WriteString(fmt.Sprintf(" Queue: %s%d image job(s) waiting%s\x1b[K\n",
|
||||||
cYellow, snap.Lock.ImageQueue, cReset))
|
cYellow, snap.Lock.ImageQueue, cReset))
|
||||||
}
|
}
|
||||||
|
if snap.GPU.Known {
|
||||||
|
b.WriteString(" GPU: " + renderVRAM(snap.GPU.UsedMB, snap.GPU.TotalMB) + "\x1b[K\n")
|
||||||
|
}
|
||||||
if snap.MonitorNote != "" {
|
if snap.MonitorNote != "" {
|
||||||
b.WriteString(" " + cYellow + snap.MonitorNote + cReset + "\x1b[K\n")
|
b.WriteString(" " + cYellow + snap.MonitorNote + cReset + "\x1b[K\n")
|
||||||
}
|
}
|
||||||
|
b.WriteString("\x1b[K\n")
|
||||||
|
b.WriteString(hotkeysLine)
|
||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// renderVRAM renders "4.2 / 16.0 GiB used" (or MiB below 1 GiB).
|
||||||
|
func renderVRAM(used, total int) string {
|
||||||
|
format := func(mb int) string {
|
||||||
|
if mb >= 1024 {
|
||||||
|
return fmt.Sprintf("%.1f GiB", float64(mb)/1024)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d MiB", mb)
|
||||||
|
}
|
||||||
|
if total > 0 {
|
||||||
|
return format(used) + " / " + format(total) + " used"
|
||||||
|
}
|
||||||
|
return format(used) + " used"
|
||||||
|
}
|
||||||
|
|
||||||
// printableLen counts characters without ANSI escapes (ASCII-only content).
|
// printableLen counts characters without ANSI escapes (ASCII-only content).
|
||||||
func printableLen(s string) int { return len(s) }
|
func printableLen(s string) int { return len(s) }
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"gpu-turnstile/internal/config"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -39,3 +40,17 @@ func TestFmtDur(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiffConfig(t *testing.T) {
|
||||||
|
a := config.Defaults()
|
||||||
|
b := a
|
||||||
|
if got := diffConfig(a, b); len(got) != 0 {
|
||||||
|
t.Fatalf("identical configs: got %v", got)
|
||||||
|
}
|
||||||
|
b.LogLevel = -4
|
||||||
|
b.GameProcs = []string{"game.exe"}
|
||||||
|
got := diffConfig(a, b)
|
||||||
|
if len(got) != 2 || got[0] != "GameProcs" || got[1] != "LogLevel" {
|
||||||
|
t.Fatalf("got %v, want [GameProcs LogLevel]", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,3 +19,21 @@ func termWidth() int {
|
|||||||
}
|
}
|
||||||
return int(ws.Col)
|
return int(ws.Col)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// enableRawKeys switches the terminal to per-keypress mode (ICANON and ECHO
|
||||||
|
// off) and returns the restore function, nil when stdin is not a terminal.
|
||||||
|
func enableRawKeys() func() {
|
||||||
|
fd := int(os.Stdin.Fd())
|
||||||
|
term, err := unix.IoctlGetTermios(fd, unix.TCGETS)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw := *term
|
||||||
|
raw.Lflag &^= unix.ICANON | unix.ECHO
|
||||||
|
raw.Cc[unix.VMIN] = 1
|
||||||
|
raw.Cc[unix.VTIME] = 0
|
||||||
|
if err := unix.IoctlSetTermios(fd, unix.TCSETS, &raw); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return func() { unix.IoctlSetTermios(fd, unix.TCSETS, term) } //nolint:errcheck
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,3 +27,23 @@ func termWidth() int {
|
|||||||
}
|
}
|
||||||
return int(info.Window.Right-info.Window.Left) + 1
|
return int(info.Window.Right-info.Window.Left) + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// enableRawKeys puts the console's stdin into per-keypress mode (no line
|
||||||
|
// buffering, no echo) and returns the restore function. When stdin is not a
|
||||||
|
// real console (mintty/Git Bash pipes) it returns nil: ptys already deliver
|
||||||
|
// keystrokes immediately.
|
||||||
|
func enableRawKeys() func() {
|
||||||
|
h := windows.Handle(os.Stdin.Fd())
|
||||||
|
var mode uint32
|
||||||
|
if err := windows.GetConsoleMode(h, &mode); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
const (
|
||||||
|
enableLineInput = 0x0002
|
||||||
|
enableEchoInput = 0x0004
|
||||||
|
)
|
||||||
|
if err := windows.SetConsoleMode(h, mode&^(enableLineInput|enableEchoInput)); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return func() { windows.SetConsoleMode(h, mode) } //nolint:errcheck
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
# ComfyUI --listen 0.0.0.0 --port 8189).
|
# ComfyUI --listen 0.0.0.0 --port 8189).
|
||||||
services:
|
services:
|
||||||
gpu-turnstile:
|
gpu-turnstile:
|
||||||
image: git.rambossek.at/public/gpu-turnstile:v0.2.7
|
image: git.rambossek.at/public/gpu-turnstile:v0.2.8
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
# Each consumer is enabled by setting its URL; leave one unset to
|
# Each consumer is enabled by setting its URL; leave one unset to
|
||||||
|
|||||||
@@ -26,6 +26,10 @@ const CmdUpdateNow = "update-now"
|
|||||||
// CmdStatus asks for a one-line JSON status snapshot (monitor mode).
|
// CmdStatus asks for a one-line JSON status snapshot (monitor mode).
|
||||||
const CmdStatus = "status"
|
const CmdStatus = "status"
|
||||||
|
|
||||||
|
// CmdReloadEnv asks the service to re-read and validate its config file,
|
||||||
|
// and to restart onto it (once the GPU is idle) when it changed.
|
||||||
|
const CmdReloadEnv = "reload-env"
|
||||||
|
|
||||||
// ErrUnavailable means no running service offers the control channel.
|
// ErrUnavailable means no running service offers the control channel.
|
||||||
var ErrUnavailable = errors.New("control channel unavailable")
|
var ErrUnavailable = errors.New("control channel unavailable")
|
||||||
|
|
||||||
|
|||||||
@@ -132,6 +132,29 @@ func queryComputeApps(ctx context.Context) ([]computeApp, error) {
|
|||||||
return parseComputeApps(string(out))
|
return parseComputeApps(string(out))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// QueryVRAMMB returns used and total GPU VRAM in MiB via nvidia-smi.
|
||||||
|
// Unlike the per-process list this works under WDDM too.
|
||||||
|
func QueryVRAMMB(ctx context.Context) (used, total int, err error) {
|
||||||
|
out, err := exec.CommandContext(ctx, "nvidia-smi",
|
||||||
|
"--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits").Output()
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
usedStr, totalStr, ok := strings.Cut(strings.TrimSpace(string(out)), ",")
|
||||||
|
if !ok {
|
||||||
|
return 0, 0, fmt.Errorf("nvidia-smi: unexpected output %q", strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
used, err = strconv.Atoi(strings.TrimSpace(usedStr))
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("nvidia-smi: unexpected used memory in %q", strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
total, err = strconv.Atoi(strings.TrimSpace(totalStr))
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("nvidia-smi: unexpected total memory in %q", strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
return used, total, nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
|
// parseComputeApps parses "pid, used_memory" CSV lines (no header, MiB
|
||||||
// units). Unsupported rows ("N/A" on WDDM) are skipped.
|
// units). Unsupported rows ("N/A" on WDDM) are skipped.
|
||||||
func parseComputeApps(out string) ([]computeApp, error) {
|
func parseComputeApps(out string) ([]computeApp, error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user