mram
bacb26772a
LLM_BUSY_MODE=reject answers blocked LLM requests immediately with LLM_BUSY_STATUS (default 503, 429 works) and Retry-After, so routers like LiteLLM can cool down and retry instead of holding a hung connection. The default wait mode now also sends Retry-After when LLM_WAIT_TIMEOUT expires. Document the service account (LocalSystem default, NT SERVICE virtual-account hardening) and the Program Files / ProgramData install layout.
35 lines
1.4 KiB
Plaintext
35 lines
1.4 KiB
Plaintext
# Example deployment for gpu-turnstile. Copy to compose.yaml and adjust.
|
|
#
|
|
# gpu-turnstile listens on the ports the services normally use; the actual
|
|
# Ollama and ComfyUI instances run one port higher (11435 / 8189) and must
|
|
# bind 0.0.0.0 so the container can reach them (OLLAMA_HOST=0.0.0.0:11435,
|
|
# ComfyUI --listen 0.0.0.0 --port 8189).
|
|
services:
|
|
gpu-turnstile:
|
|
image: git.rambossek.at/public/gpu-turnstile:v0.1.2
|
|
restart: unless-stopped
|
|
environment:
|
|
# Services on the Docker host itself:
|
|
OLLAMA_URL: http://host.docker.internal:11435
|
|
COMFY_URL: http://host.docker.internal:8189
|
|
# Services on another machine: use its LAN IP instead, e.g.
|
|
# OLLAMA_URL: http://192.168.1.10:11435
|
|
# COMFY_URL: http://192.168.1.10:8189
|
|
# UNLOAD_TIMEOUT: 60s
|
|
# JOB_TIMEOUT: 15m
|
|
# LLM_WAIT_TIMEOUT: 10m
|
|
# LLM_BUSY_MODE: reject # wait (default) hangs; reject fails fast
|
|
# LLM_BUSY_STATUS: 429 # status in reject mode (default 503)
|
|
# BUSY_RETRY_AFTER: 30 # Retry-After seconds on busy responses
|
|
# WARM_MODEL: qwen3:14b
|
|
# LOG_LEVEL: info
|
|
# LOG_FORMAT: json
|
|
ports:
|
|
- "11434:11434" # LiteLLM api_base -> http://gpu-turnstile:11434
|
|
- "8188:8188" # Open WebUI COMFYUI_BASE_URL -> http://gpu-turnstile:8188
|
|
networks: [internal]
|
|
|
|
networks:
|
|
internal:
|
|
external: true
|