Add LLM busy modes: wait (hang) or reject with Retry-After
LLM_BUSY_MODE=reject answers blocked LLM requests immediately with LLM_BUSY_STATUS (default 503, 429 works) and Retry-After, so routers like LiteLLM can cool down and retry instead of holding a hung connection. The default wait mode now also sends Retry-After when LLM_WAIT_TIMEOUT expires. Document the service account (LocalSystem default, NT SERVICE virtual-account hardening) and the Program Files / ProgramData install layout.
This commit is contained in:
@@ -74,6 +74,22 @@ func (l *Lock) AcquireLLM(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TryAcquireLLM acquires one in-flight LLM slot without waiting and
|
||||
// reports whether it succeeded. It fails when an image job is active or
|
||||
// pending.
|
||||
func (l *Lock) TryAcquireLLM() bool {
|
||||
l.mu.Lock()
|
||||
if l.imageActive || len(l.imageQ) > 0 {
|
||||
l.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
l.n++
|
||||
n := l.n
|
||||
l.mu.Unlock()
|
||||
l.logTransition("lock transition", "state", StateLLM, "llm_inflight", n)
|
||||
return true
|
||||
}
|
||||
|
||||
// ReleaseLLM marks one LLM request as finished.
|
||||
func (l *Lock) ReleaseLLM() {
|
||||
l.mu.Lock()
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package lock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTryAcquireLLM(t *testing.T) {
|
||||
lk := New(nil)
|
||||
if !lk.TryAcquireLLM() {
|
||||
t.Fatal("TryAcquireLLM on idle lock should succeed")
|
||||
}
|
||||
if _, n, _ := lk.Snapshot(); n != 1 {
|
||||
t.Fatalf("n = %d, want 1", n)
|
||||
}
|
||||
|
||||
// While an image job is pending, TryAcquireLLM must fail.
|
||||
imageWaiting := make(chan struct{})
|
||||
go func() {
|
||||
lk.AcquireImage(context.Background())
|
||||
close(imageWaiting)
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
lk.mu.Lock()
|
||||
queued := len(lk.imageQ)
|
||||
lk.mu.Unlock()
|
||||
if queued == 1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("image waiter never queued")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if lk.TryAcquireLLM() {
|
||||
t.Fatal("TryAcquireLLM with image pending should fail")
|
||||
}
|
||||
|
||||
lk.ReleaseLLM()
|
||||
<-imageWaiting
|
||||
if lk.TryAcquireLLM() {
|
||||
t.Fatal("TryAcquireLLM with image active should fail")
|
||||
}
|
||||
lk.ReleaseImage()
|
||||
if !lk.TryAcquireLLM() {
|
||||
t.Fatal("TryAcquireLLM after image release should succeed")
|
||||
}
|
||||
lk.ReleaseLLM()
|
||||
}
|
||||
Reference in New Issue
Block a user