Add LLM busy modes: wait (hang) or reject with Retry-After

LLM_BUSY_MODE=reject answers blocked LLM requests immediately with
LLM_BUSY_STATUS (default 503, 429 works) and Retry-After, so routers
like LiteLLM can cool down and retry instead of holding a hung
connection. The default wait mode now also sends Retry-After when
LLM_WAIT_TIMEOUT expires. Document the service account (LocalSystem
default, NT SERVICE virtual-account hardening) and the Program
Files / ProgramData install layout.
This commit is contained in:
mram
2026-09-20 21:43:03 +02:00
parent e363c9e4e4
commit bacb26772a
10 changed files with 291 additions and 3 deletions
+16
View File
@@ -74,6 +74,22 @@ func (l *Lock) AcquireLLM(ctx context.Context) error {
return nil
}
// TryAcquireLLM acquires one in-flight LLM slot without waiting and
// reports whether it succeeded. It fails when an image job is active or
// pending.
func (l *Lock) TryAcquireLLM() bool {
l.mu.Lock()
if l.imageActive || len(l.imageQ) > 0 {
l.mu.Unlock()
return false
}
l.n++
n := l.n
l.mu.Unlock()
l.logTransition("lock transition", "state", StateLLM, "llm_inflight", n)
return true
}
// ReleaseLLM marks one LLM request as finished.
func (l *Lock) ReleaseLLM() {
l.mu.Lock()