Add LLM busy modes: wait (hang) or reject with Retry-After
LLM_BUSY_MODE=reject answers blocked LLM requests immediately with LLM_BUSY_STATUS (default 503, 429 works) and Retry-After, so routers like LiteLLM can cool down and retry instead of holding a hung connection. The default wait mode now also sends Retry-After when LLM_WAIT_TIMEOUT expires. Document the service account (LocalSystem default, NT SERVICE virtual-account hardening) and the Program Files / ProgramData install layout.
This commit is contained in:
@@ -74,6 +74,22 @@ func (l *Lock) AcquireLLM(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TryAcquireLLM acquires one in-flight LLM slot without waiting and
|
||||
// reports whether it succeeded. It fails when an image job is active or
|
||||
// pending.
|
||||
func (l *Lock) TryAcquireLLM() bool {
|
||||
l.mu.Lock()
|
||||
if l.imageActive || len(l.imageQ) > 0 {
|
||||
l.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
l.n++
|
||||
n := l.n
|
||||
l.mu.Unlock()
|
||||
l.logTransition("lock transition", "state", StateLLM, "llm_inflight", n)
|
||||
return true
|
||||
}
|
||||
|
||||
// ReleaseLLM marks one LLM request as finished.
|
||||
func (l *Lock) ReleaseLLM() {
|
||||
l.mu.Lock()
|
||||
|
||||
Reference in New Issue
Block a user