mirror of
https://github.com/nomic-ai/gpt4all
synced 2024-11-06 09:20:33 +00:00
24 lines
615 B
YAML
24 lines
615 B
YAML
|
version: "3.8"
|
||
|
|
|
||
|
|
services:
|
||
|
|
gpt4all_gpu:
|
||
|
|
image: ghcr.io/huggingface/text-generation-inference
|
||
|
|
container_name: gpt4all_gpu
|
||
|
|
restart: always #restart on error (usually code compilation from save during bad state)
|
||
|
|
environment:
|
||
|
|
- HUGGING_FACE_HUB_TOKEN=token
|
||
|
|
- USE_FLASH_ATTENTION=false
|
||
|
|
- MODEL_ID=''
|
||
|
|
- NUM_SHARD=1
|
||
|
|
command: --model-id $MODEL_ID --num-shard $NUM_SHARD
|
||
|
|
volumes:
|
||
|
|
- ./:/data
|
||
|
|
ports:
|
||
|
|
- "8080:80"
|
||
|
|
shm_size: 1g
|
||
|
|
deploy:
|
||
|
|
resources:
|
||
|
|
reservations:
|
||
|
|
devices:
|
||
|
|
- driver: nvidia
|
||
|
|
capabilities: [gpu]
|