feat(voice): add OpenVINO NPU Whisper service
This commit is contained in:
William Valentin
+150
-17
@@ -30,31 +30,166 @@ services:
|
||||
# start_period: 15s
|
||||
# retries: 3
|
||||
|
||||
# Optional local dependency: whisper.cpp server for audio transcription.
|
||||
# Start with: docker compose --profile voice up -d whisper-server
|
||||
whisper-server:
|
||||
image: ghcr.io/ggml-org/whisper.cpp@sha256:3a39e86d5a0e911086b5cbebc9029cac71b02fbd08e217b775857de1490f55bf
|
||||
container_name: whisper-server
|
||||
# One-shot init: download whisper models into the shared volume if missing.
|
||||
# The base image only ships ggml-base.en.bin; the servers below require:
|
||||
# - ggml-medium.bin for the CPU server
|
||||
# - ggml-small.bin for the GPU server (small fits in the limited VRAM left after gemma)
|
||||
whisper-init:
|
||||
image: ghcr.io/ggml-org/whisper.cpp@sha256:672650b5e67f9cb86af7ac6e09dea8eac12a024086e1e5c0172fdccf336aba09
|
||||
container_name: whisper-init
|
||||
profiles: ["voice", "voice-cpu-backup"]
|
||||
restart: "no"
|
||||
volumes:
|
||||
- whisper-models:/app/models
|
||||
entrypoint: ["sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -e
|
||||
for m in medium small base; do
|
||||
if [ -f /app/models/ggml-$$m.bin ]; then
|
||||
echo "Model ggml-$$m.bin already present, skipping download."
|
||||
else
|
||||
echo "Downloading ggml-$$m.bin..."
|
||||
sh /app/models/download-ggml-model.sh $$m /app/models
|
||||
fi
|
||||
done
|
||||
|
||||
# Manual GPU whisper.cpp fallback: NVIDIA RTX 5070 Ti via CUDA (Blackwell sm_120).
|
||||
# Kept out of the normal `voice` profile because the OpenVINO NPU Whisper
|
||||
# service is the default and this container consumes GPU resources.
|
||||
#
|
||||
# The official `ghcr.io/ggml-org/whisper.cpp:main-cuda` ships kernels only
|
||||
# for sm_75/80/86/90 and fails to init CUDA on Blackwell. We build a custom
|
||||
# image with `CMAKE_CUDA_ARCHITECTURES=120` from the local Dockerfile.
|
||||
# Build manually with: docker build -t whisper.cpp:cuda-blackwell ./whisper-cuda-blackwell
|
||||
# Or `docker compose --profile voice-gpu build whisper-server-gpu`.
|
||||
whisper-server-gpu:
|
||||
image: whisper.cpp:cuda-blackwell
|
||||
build:
|
||||
context: ./whisper-cuda-blackwell
|
||||
dockerfile: Dockerfile
|
||||
container_name: whisper-server-gpu
|
||||
restart: unless-stopped
|
||||
profiles: ["voice"]
|
||||
profiles: ["voice-gpu"]
|
||||
ports:
|
||||
- "18801:8080"
|
||||
volumes:
|
||||
- whisper-models:/app/models
|
||||
# Override image entrypoint so args are passed directly to whisper-server.
|
||||
entrypoint: ["whisper-server"]
|
||||
command:
|
||||
- --model
|
||||
- /app/models/ggml-base.en.bin
|
||||
- /app/models/ggml-base.bin
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8080"
|
||||
- --convert
|
||||
- --language
|
||||
- en
|
||||
- auto
|
||||
- --inference-path
|
||||
- /v1/audio/transcriptions
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
depends_on:
|
||||
whisper-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"curl -f http://localhost:8080/ >/dev/null 2>&1 || exit 1",
|
||||
]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
start_period: 30s
|
||||
retries: 3
|
||||
labels:
|
||||
agentmon.monitor: "true"
|
||||
agentmon.role: "voice"
|
||||
agentmon.port: "18801"
|
||||
|
||||
# Experimental OpenVINO GenAI Whisper server using the Intel NPU.
|
||||
# This is not whisper.cpp; it implements the same OpenAI-style
|
||||
# /v1/audio/transcriptions route using OpenVINO WhisperPipeline on NPU.
|
||||
# Host requirements: intel-npu-driver-bin installed, /dev/accel/accel0 present,
|
||||
# and the host NPU Level Zero driver/compiler libraries mounted below.
|
||||
whisper-server-npu:
|
||||
image: whisper-openvino-npu:local
|
||||
build:
|
||||
context: ./whisper-openvino-npu
|
||||
dockerfile: Dockerfile
|
||||
container_name: whisper-server-npu
|
||||
restart: unless-stopped
|
||||
profiles: ["voice"]
|
||||
ports:
|
||||
- "18816:8080"
|
||||
devices:
|
||||
- /dev/accel/accel0:/dev/accel/accel0
|
||||
group_add:
|
||||
- "987" # host render group gid on willlaptop
|
||||
environment:
|
||||
- WHISPER_DEVICE=NPU
|
||||
- WHISPER_MODEL_DIR=/models/whisper-tiny-fp16-ov
|
||||
- LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu
|
||||
- ZE_ENABLE_ALT_DRIVERS=/usr/lib/x86_64-linux-gnu/libze_intel_npu.so.1
|
||||
volumes:
|
||||
- /home/will/.cache/openvino-models/whisper-tiny-fp16-ov:/models/whisper-tiny-fp16-ov:ro
|
||||
- /usr/lib/x86_64-linux-gnu/libze_intel_npu.so.1.32.1:/usr/lib/x86_64-linux-gnu/libze_intel_npu.so.1.32.1:ro
|
||||
- /usr/lib/x86_64-linux-gnu/libze_intel_npu.so.1:/usr/lib/x86_64-linux-gnu/libze_intel_npu.so.1:ro
|
||||
- /usr/lib/x86_64-linux-gnu/libze_intel_npu.so:/usr/lib/x86_64-linux-gnu/libze_intel_npu.so:ro
|
||||
- /usr/lib/x86_64-linux-gnu/libnpu_driver_compiler.so:/usr/lib/x86_64-linux-gnu/libnpu_driver_compiler.so:ro
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"curl -f http://localhost:8080/health >/dev/null 2>&1 || exit 1",
|
||||
]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
start_period: 30s
|
||||
retries: 3
|
||||
labels:
|
||||
agentmon.monitor: "true"
|
||||
agentmon.role: "voice"
|
||||
agentmon.port: "18816"
|
||||
|
||||
# Manual fallback whisper.cpp server: CPU-only, medium model.
|
||||
# Kept around for resilience — runs if the NPU/GPU servers are down. Uses no
|
||||
# accelerator resources, but is slow (~14 s per short clip).
|
||||
# Disabled from the normal `voice` profile now that `whisper-server-npu` is
|
||||
# the trial default. Start manually with:
|
||||
# docker compose --profile voice-cpu-backup up -d whisper-server
|
||||
whisper-server:
|
||||
image: ghcr.io/ggml-org/whisper.cpp@sha256:672650b5e67f9cb86af7ac6e09dea8eac12a024086e1e5c0172fdccf336aba09
|
||||
container_name: whisper-server
|
||||
restart: unless-stopped
|
||||
profiles: ["voice-cpu-backup"]
|
||||
ports:
|
||||
- "18811:8080"
|
||||
volumes:
|
||||
- whisper-models:/app/models
|
||||
# Override image entrypoint so args are passed directly to whisper-server.
|
||||
entrypoint: ["whisper-server"]
|
||||
command:
|
||||
- --model
|
||||
- /app/models/ggml-medium.bin
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8080"
|
||||
- --convert
|
||||
- --language
|
||||
- auto
|
||||
- --inference-path
|
||||
- /v1/audio/transcriptions
|
||||
depends_on:
|
||||
whisper-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
@@ -68,7 +203,7 @@ services:
|
||||
labels:
|
||||
agentmon.monitor: "true"
|
||||
agentmon.role: "voice"
|
||||
agentmon.port: "18801"
|
||||
agentmon.port: "18811"
|
||||
|
||||
# kokoro TTS
|
||||
kokoro-tts:
|
||||
@@ -134,7 +269,7 @@ services:
|
||||
# Optional local dependency: liteLLM proxy for unified LLM API.
|
||||
# Start with: docker compose --profile api up -d litellm
|
||||
litellm:
|
||||
image: litellm/litellm:v1.82.3-stable.patch.2
|
||||
image: litellm/litellm:v1.83.7-stable
|
||||
container_name: litellm
|
||||
restart: unless-stopped
|
||||
profiles: ["api"]
|
||||
@@ -142,7 +277,6 @@ services:
|
||||
- "18804:4000"
|
||||
volumes:
|
||||
- ./litellm-config.yaml:/app/config.yaml:ro
|
||||
- ./litellm-copilot-tokens:/root/.config/litellm/github_copilot
|
||||
environment:
|
||||
- LITELLM_PORT=4000
|
||||
- LITELLM_DROP_PARAMS=true
|
||||
@@ -151,7 +285,6 @@ services:
|
||||
- OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-}
|
||||
- GEMINI_API_KEY=${GEMINI_API_KEY:-}
|
||||
- ZAI_API_KEY=${ZAI_API_KEY:-}
|
||||
- GITHUB_COPILOT_TOKEN_DIR=/root/.config/litellm/github_copilot
|
||||
- DATABASE_URL=postgresql://litellm:litellm_password@litellm-db:5432/litellm
|
||||
- LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY:-sk-1234}
|
||||
- LITELLM_SALT_KEY=${LITELLM_SALT_KEY:-}
|
||||
@@ -198,7 +331,7 @@ services:
|
||||
condition: service_healthy
|
||||
|
||||
litellm-db:
|
||||
image: postgres:15-alpine
|
||||
image: postgres:15.17-alpine
|
||||
container_name: litellm-db
|
||||
restart: unless-stopped
|
||||
profiles: ["api"]
|
||||
@@ -221,7 +354,7 @@ services:
|
||||
# Dedicated local n8n instance for agent-oriented workflows.
|
||||
# Start with: docker compose --profile automation up -d n8n-agent
|
||||
n8n-agent:
|
||||
image: docker.n8n.io/n8nio/n8n:2.11.3
|
||||
image: docker.n8n.io/n8nio/n8n:2.22.1
|
||||
container_name: n8n-agent
|
||||
restart: unless-stopped
|
||||
profiles: ["automation"]
|
||||
@@ -233,8 +366,8 @@ services:
|
||||
- N8N_PROTOCOL=http
|
||||
- N8N_EDITOR_BASE_URL=http://localhost:18808
|
||||
- WEBHOOK_URL=http://localhost:18808/
|
||||
- TZ=UTC
|
||||
- GENERIC_TIMEZONE=UTC
|
||||
- TZ=America/Los_Angeles
|
||||
- GENERIC_TIMEZONE=America/Los_Angeles
|
||||
- N8N_SECURE_COOKIE=false
|
||||
volumes:
|
||||
- n8n-agent-data:/home/node/.n8n
|
||||
|
||||
Reference in New Issue
Block a user