Add LLM busy modes: wait (hang) or reject with Retry-After

LLM_BUSY_MODE=reject answers blocked LLM requests immediately with
LLM_BUSY_STATUS (default 503, 429 works) and Retry-After, so routers
like LiteLLM can cool down and retry instead of holding a hung
connection. The default wait mode now also sends Retry-After when
LLM_WAIT_TIMEOUT expires. Document the service account (LocalSystem
default, NT SERVICE virtual-account hardening) and the Program
Files / ProgramData install layout.
This commit is contained in:
mram
2026-09-20 21:43:03 +02:00
parent e363c9e4e4
commit bacb26772a
10 changed files with 291 additions and 3 deletions
+6
View File
@@ -192,6 +192,9 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
"unload_timeout", cfg.UnloadTimeout,
"job_timeout", cfg.JobTimeout,
"llm_wait_timeout", cfg.LLMWaitTimeout,
"llm_busy_mode", cfg.LLMBusyMode,
"llm_busy_status", cfg.LLMBusyStatus,
"busy_retry_after", cfg.BusyRetryAfter,
"unload_poll_interval", cfg.UnloadPollInterval,
"history_poll_interval", cfg.HistoryPollInterval,
"probe_timeout", cfg.ProbeTimeout,
@@ -238,6 +241,9 @@ func run(ctx context.Context, cfg config.Config, log *slog.Logger, logOut io.Wri
HistoryPollInterval: cfg.HistoryPollInterval,
FreeTimeout: cfg.FreeTimeout,
WarmTimeout: cfg.WarmTimeout,
LLMBusyMode: cfg.LLMBusyMode,
LLMBusyStatus: cfg.LLMBusyStatus,
BusyRetryAfter: cfg.BusyRetryAfter,
BackoffInitial: cfg.BackoffInitial,
BackoffMax: cfg.BackoffMax,
PromptCaptureLimit: cfg.PromptCaptureLimit,