Local control channel: unprivileged users can trigger --force-update via the running service
This commit is contained in:
+92
-22
@@ -17,11 +17,13 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gpu-turnstile/internal/comfy"
|
||||
"gpu-turnstile/internal/config"
|
||||
"gpu-turnstile/internal/control"
|
||||
"gpu-turnstile/internal/game"
|
||||
"gpu-turnstile/internal/lock"
|
||||
"gpu-turnstile/internal/metrics"
|
||||
@@ -98,6 +100,7 @@ Usage:
|
||||
gpu-turnstile -v | --version print just the version
|
||||
gpu-turnstile --force-update check for a signed update now,
|
||||
apply it and restart the service
|
||||
(no admin needed when the service runs)
|
||||
gpu-turnstile -h | --help this help
|
||||
|
||||
Options:
|
||||
@@ -347,6 +350,13 @@ func forceUpdateCommand(configPath string, elevatedChild bool) int {
|
||||
fmt.Printf("%s: APP_VER=dev, updates disabled\n", versionLine())
|
||||
return 0
|
||||
}
|
||||
// A running service can do the privileged work itself (its account owns
|
||||
// the install dir and it knows when the GPU is idle): ask it over the
|
||||
// local control channel first, no admin rights needed. Fails fast when
|
||||
// no service is listening, in which case we do the direct check below.
|
||||
if reply, err := control.Ask(control.CmdUpdateNow); err == nil {
|
||||
return printControlReply(reply)
|
||||
}
|
||||
u := &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log}
|
||||
|
||||
// Single-shot: one attempt, fail fast when the server is unreachable
|
||||
@@ -397,6 +407,17 @@ func forceUpdateCommand(configPath string, elevatedChild bool) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// printControlReply prints the service's answer to a control-channel
|
||||
// request: "OK ..." on stdout (exit 0), "ERR ..." on stderr (exit 1).
|
||||
func printControlReply(reply string) int {
|
||||
if msg, ok := strings.CutPrefix(reply, "ERR "); ok {
|
||||
fmt.Fprintf(os.Stderr, "gpu-turnstile: %s\n", msg)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("gpu-turnstile: " + strings.TrimPrefix(reply, "OK "))
|
||||
return 0
|
||||
}
|
||||
|
||||
// reportElevatedUpdate prints the parent's summary of an elevated
|
||||
// --force-update child: exitCodeStaged means the child staged a new binary,
|
||||
// 0 means it found nothing to do. to is the tag the parent's own check
|
||||
@@ -688,7 +709,35 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
|
||||
service.StartWatchdog(ctx)
|
||||
|
||||
if cfg.AutoUpdate {
|
||||
go updateLoop(ctx, cfg, log, lk, isService)
|
||||
exePath, err := os.Executable()
|
||||
if err != nil {
|
||||
log.Warn("auto-update disabled: cannot locate executable", "err", err)
|
||||
} else {
|
||||
u := &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log}
|
||||
// applyStaged is shared by the hourly loop and the control
|
||||
// channel; the once guard keeps a second trigger from
|
||||
// double-waiting on the GPU lock.
|
||||
var once sync.Once
|
||||
applyStaged := func(to string) {
|
||||
if !isService {
|
||||
log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply", "version", to)
|
||||
return
|
||||
}
|
||||
log.Warn("auto-update: staged; restarting once the GPU is idle", "version", to)
|
||||
once.Do(func() {
|
||||
go func() {
|
||||
if waitForIdle(ctx, lk, 24*time.Hour) {
|
||||
log.Warn("auto-update: restarting to apply update")
|
||||
os.Exit(exitCodeUpdate)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
go updateLoop(ctx, cfg.UpdateInterval, log, u, exePath, applyStaged)
|
||||
if isService {
|
||||
serveControl(ctx, log, u, exePath, applyStaged)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -830,42 +879,63 @@ func healthLoop(ctx context.Context, interval, probeTimeout time.Duration, log *
|
||||
}
|
||||
}
|
||||
|
||||
// updateLoop checks for signed updates on startup and every UPDATE_INTERVAL.
|
||||
// In service mode a staged update is applied by exiting with exitCodeUpdate
|
||||
// once the GPU lock is idle; the service recovery configuration restarts the
|
||||
// process with the new binary. Interactively it only logs.
|
||||
func updateLoop(ctx context.Context, cfg config.Config, log *slog.Logger, lk *lock.Lock, isService bool) {
|
||||
exePath, err := os.Executable()
|
||||
if err != nil {
|
||||
log.Warn("auto-update disabled: cannot locate executable", "err", err)
|
||||
return
|
||||
}
|
||||
u := &update.Updater{Repo: cfg.UpdateRepo, Asset: cfg.UpdateAsset, Version: version, Desired: cfg.AppVersion, Log: log}
|
||||
// updateLoop checks for signed updates on startup and every interval;
|
||||
// applyStaged decides what a staged update means (restart when idle as a
|
||||
// service, log only interactively).
|
||||
func updateLoop(ctx context.Context, interval time.Duration, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string)) {
|
||||
for {
|
||||
staged, to, err := u.Check(ctx, exePath)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
log.Warn("auto-update check failed", "err", err)
|
||||
}
|
||||
if staged {
|
||||
if !isService {
|
||||
log.Warn("auto-update: new binary staged; restart gpu-turnstile to apply", "version", to)
|
||||
return
|
||||
}
|
||||
log.Warn("auto-update: staged; restarting once the GPU is idle", "version", to)
|
||||
if waitForIdle(ctx, lk, 24*time.Hour) {
|
||||
log.Warn("auto-update: restarting to apply update")
|
||||
os.Exit(exitCodeUpdate)
|
||||
}
|
||||
applyStaged(to)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(cfg.UpdateInterval):
|
||||
case <-time.After(interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// serveControl opens the local control channel (named pipe on Windows,
|
||||
// unix socket on Linux) so unprivileged local users can trigger an update
|
||||
// check via --force-update without admin rights. The check payload is
|
||||
// signature-verified regardless of who asks; triggers are rate-limited to
|
||||
// one per minute so the channel cannot be used to spam restarts.
|
||||
func serveControl(ctx context.Context, log *slog.Logger, u *update.Updater, exePath string, applyStaged func(to string)) {
|
||||
var mu sync.Mutex
|
||||
var lastTrigger time.Time
|
||||
h := func(cmd string) string {
|
||||
if cmd != control.CmdUpdateNow {
|
||||
return "ERR unknown command: " + cmd
|
||||
}
|
||||
mu.Lock()
|
||||
if wait := time.Minute - time.Since(lastTrigger); wait > 0 {
|
||||
mu.Unlock()
|
||||
return fmt.Sprintf("ERR rate limited: retry in %ds", int(wait.Seconds())+1)
|
||||
}
|
||||
lastTrigger = time.Now()
|
||||
mu.Unlock()
|
||||
cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
staged, to, err := u.Check(cctx, exePath)
|
||||
if err != nil {
|
||||
return "ERR update check failed: " + err.Error()
|
||||
}
|
||||
if !staged {
|
||||
return "OK " + version + " is up to date"
|
||||
}
|
||||
applyStaged(to)
|
||||
return "OK updated from " + version + " to " + to + "; the service restarts once the GPU is idle"
|
||||
}
|
||||
if err := control.Serve(ctx, h, log); err != nil {
|
||||
log.Warn("control channel disabled", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForIdle polls the lock until no LLM or image work is active or
|
||||
// pending, max at most. Returns false on timeout or cancellation.
|
||||
func waitForIdle(ctx context.Context, lk *lock.Lock, max time.Duration) bool {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
// Package control exposes a local-only command channel into a running
|
||||
// gpu-turnstile service: a named pipe on Windows, a unix socket on Linux.
|
||||
// It lets unprivileged local users ask the service to do privileged work
|
||||
// that is safe to offer — currently triggering an update check, whose
|
||||
// payload is signature-verified regardless of who asks. The channel never
|
||||
// accepts data beyond a one-word command, and the server rate-limits
|
||||
// triggers, so the worst a local user can cause is a cheap, throttled
|
||||
// check and a GPU-idle-gated restart onto a signed binary.
|
||||
//
|
||||
// Protocol: the client writes one command line, the server answers with
|
||||
// one reply line ("OK ..." or "ERR ...") and hangs up.
|
||||
package control
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CmdUpdateNow asks the service to check for, stage and (once the GPU is
|
||||
// idle) restart onto a signed update immediately.
|
||||
const CmdUpdateNow = "update-now"
|
||||
|
||||
// ErrUnavailable means no running service offers the control channel.
|
||||
var ErrUnavailable = errors.New("control channel unavailable")
|
||||
|
||||
// Handler answers one command; the returned string is sent back as one
|
||||
// line. It must start with "OK " or "ERR ".
|
||||
type Handler func(cmd string) string
|
||||
|
||||
// serveConn runs the line protocol on one accepted connection.
|
||||
func serveConn(c io.ReadWriteCloser, h Handler) {
|
||||
defer c.Close()
|
||||
line, err := bufio.NewReader(io.LimitReader(c, 4096)).ReadString('\n')
|
||||
cmd := strings.TrimSpace(line)
|
||||
if cmd == "" {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(c, "ERR empty command")
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(c, h(cmd))
|
||||
}
|
||||
|
||||
// readReply writes cmd and reads the server's one-line reply.
|
||||
func readReply(c io.ReadWriteCloser, cmd string) (string, error) {
|
||||
if _, err := fmt.Fprintln(c, cmd); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The server hangs up after its reply; a broken-pipe error after the
|
||||
// last byte still leaves the reply in the buffer.
|
||||
data, _ := io.ReadAll(io.LimitReader(c, 4096))
|
||||
line := strings.TrimSpace(string(data))
|
||||
if line == "" {
|
||||
return "", ErrUnavailable
|
||||
}
|
||||
return line, nil
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//go:build linux
|
||||
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// sockPath lives in the unit's RuntimeDirectory; mode 0666 lets every
|
||||
// local user ask, nothing can reach it from off the machine.
|
||||
const sockPath = "/run/gpu-turnstile/control.sock"
|
||||
|
||||
// Serve starts the socket listener in the background and returns; only a
|
||||
// setup failure is reported. Each client connection is answered in its own
|
||||
// goroutine.
|
||||
func Serve(ctx context.Context, h Handler, log *slog.Logger) error {
|
||||
os.Remove(sockPath) // stale socket from a previous run
|
||||
ln, err := net.Listen("unix", sockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(sockPath, 0o666); err != nil {
|
||||
ln.Close()
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
ln.Close()
|
||||
}()
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return // shutting down
|
||||
}
|
||||
go serveConn(c, h)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ask sends one command to the running service and returns its reply.
|
||||
func Ask(cmd string) (string, error) {
|
||||
c, err := net.DialTimeout("unix", sockPath, 2*time.Second)
|
||||
if err != nil {
|
||||
return "", ErrUnavailable
|
||||
}
|
||||
defer c.Close()
|
||||
return readReply(c, cmd)
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//go:build !windows && !linux
|
||||
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// Serve is a no-op on platforms without a control channel.
|
||||
func Serve(ctx context.Context, h Handler, log *slog.Logger) error {
|
||||
return ErrUnavailable
|
||||
}
|
||||
|
||||
// Ask always reports the channel as unavailable.
|
||||
func Ask(cmd string) (string, error) {
|
||||
return "", ErrUnavailable
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
server, client := net.Pipe()
|
||||
go serveConn(server, func(cmd string) string {
|
||||
if cmd != CmdUpdateNow {
|
||||
return "ERR unknown command: " + cmd
|
||||
}
|
||||
return "OK v0.2.2 is up to date"
|
||||
})
|
||||
reply, err := readReply(client, CmdUpdateNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reply != "OK v0.2.2 is up to date" {
|
||||
t.Fatalf("reply = %q", reply)
|
||||
}
|
||||
|
||||
server2, client2 := net.Pipe()
|
||||
go serveConn(server2, func(cmd string) string { return "ERR unknown command: " + cmd })
|
||||
reply, err = readReply(client2, "bogus")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(reply, "ERR ") {
|
||||
t.Fatalf("reply = %q, want ERR prefix", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyReplyIsUnavailable(t *testing.T) {
|
||||
server, client := net.Pipe()
|
||||
go serveConn(server, func(cmd string) string {
|
||||
server.Close() // hang up without answering
|
||||
return ""
|
||||
})
|
||||
if _, err := readReply(client, CmdUpdateNow); !errors.Is(err, ErrUnavailable) {
|
||||
t.Fatalf("err = %v, want ErrUnavailable", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
//go:build windows
|
||||
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// pipePath is a kernel-local named pipe: no TCP, no firewall prompt.
|
||||
const pipePath = `\\.\pipe\gpu-turnstile`
|
||||
|
||||
// sddlPipe grants full access to Administrators, SYSTEM and the pipe owner,
|
||||
// and read+write to authenticated users — except network logons, so the
|
||||
// pipe cannot be reached from another machine over SMB.
|
||||
const sddlPipe = "D:(D;;GRGW;;;NU)(A;;GA;;;BA)(A;;GA;;;SY)(A;;GA;;;OW)(A;;GRGW;;;AU)"
|
||||
|
||||
var (
|
||||
procConvertSDDL = windows.NewLazySystemDLL("advapi32.dll").
|
||||
NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW")
|
||||
procWaitNamedPipe = windows.NewLazySystemDLL("kernel32.dll").
|
||||
NewProc("WaitNamedPipeW")
|
||||
)
|
||||
|
||||
func waitNamedPipe(name *uint16, timeout uint32) error {
|
||||
r, _, err := procWaitNamedPipe.Call(uintptr(unsafe.Pointer(name)), uintptr(timeout))
|
||||
if r == 0 {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func securityAttributesFromSDDL(sddl string) (*windows.SecurityAttributes, error) {
|
||||
s, err := windows.UTF16PtrFromString(sddl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sd *uint16 // SECURITY_DESCRIPTOR*, kept for the process lifetime
|
||||
r, _, callErr := procConvertSDDL.Call(
|
||||
uintptr(unsafe.Pointer(s)), 1, /* SDDL_REVISION_1 */
|
||||
uintptr(unsafe.Pointer(&sd)), 0)
|
||||
if r == 0 {
|
||||
return nil, fmt.Errorf("invalid SDDL: %w", callErr)
|
||||
}
|
||||
sa := &windows.SecurityAttributes{
|
||||
Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})),
|
||||
SecurityDescriptor: (*windows.SECURITY_DESCRIPTOR)(unsafe.Pointer(sd)),
|
||||
}
|
||||
return sa, nil
|
||||
}
|
||||
|
||||
// Serve starts the pipe listener in the background and returns; only a
|
||||
// setup failure is reported. Each client connection is answered in its own
|
||||
// goroutine. On shutdown the process exit reaps everything.
|
||||
func Serve(ctx context.Context, h Handler, log *slog.Logger) error {
|
||||
sa, err := securityAttributesFromSDDL(sddlPipe)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name, err := windows.UTF16PtrFromString(pipePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
for ctx.Err() == nil {
|
||||
pipe, err := windows.CreateNamedPipe(name,
|
||||
windows.PIPE_ACCESS_DUPLEX,
|
||||
windows.PIPE_TYPE_BYTE|windows.PIPE_READMODE_BYTE|windows.PIPE_WAIT,
|
||||
16, 4096, 4096, 0, sa)
|
||||
if err != nil {
|
||||
log.Warn("control channel stopped", "err", err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
// Blocks until a client connects; on process exit the
|
||||
// handle goes away with everything else.
|
||||
if err := windows.ConnectNamedPipe(pipe, nil); err != nil {
|
||||
windows.CloseHandle(pipe)
|
||||
return
|
||||
}
|
||||
f := os.NewFile(uintptr(pipe), pipePath)
|
||||
serveConn(f, h) // closes f, and with it the pipe handle
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ask sends one command to the running service and returns its reply.
|
||||
func Ask(cmd string) (string, error) {
|
||||
name, err := windows.UTF16PtrFromString(pipePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := waitNamedPipe(name, 2000); err != nil {
|
||||
return "", ErrUnavailable
|
||||
}
|
||||
handle, err := windows.CreateFile(name,
|
||||
windows.GENERIC_READ|windows.GENERIC_WRITE, 0, nil,
|
||||
windows.OPEN_EXISTING, 0, 0)
|
||||
if err != nil {
|
||||
return "", ErrUnavailable
|
||||
}
|
||||
f := os.NewFile(uintptr(handle), pipePath)
|
||||
defer f.Close()
|
||||
return readReply(f, cmd)
|
||||
}
|
||||
@@ -95,6 +95,8 @@ RestartSec=5s
|
||||
|
||||
DynamicUser=yes
|
||||
StateDirectory=%s
|
||||
RuntimeDirectory=%s
|
||||
RuntimeDirectoryMode=0755
|
||||
%sProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
@@ -117,7 +119,7 @@ SystemCallErrorNumber=EPERM
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`, exePath, configPath, Name, bind)
|
||||
`, exePath, configPath, Name, Name, bind)
|
||||
}
|
||||
|
||||
// copyFile copies src to dst, creating dst with the given mode.
|
||||
|
||||
@@ -17,6 +17,7 @@ func TestRenderUnit(t *testing.T) {
|
||||
"WantedBy=multi-user.target",
|
||||
"DynamicUser=yes",
|
||||
"StateDirectory=gpu-turnstile",
|
||||
"RuntimeDirectory=gpu-turnstile",
|
||||
"ProtectSystem=strict",
|
||||
"NoNewPrivileges=yes",
|
||||
"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6",
|
||||
|
||||
Reference in New Issue
Block a user