简介

• 本文介绍一下 ktransfomers 单机部署depseek 满血版
• 项目：https://github.com/kvcache-ai/ktransformers

镜像构建

• Dockerfile

FROM approachingai/ktransformers:0.2.1

RUN wget https://github.com/kvcache-ai/ktransformers/releases/download/v0.2.0/ktransformers-0.2.0+cu121torch23fancy-cp310-cp310-linux_x86_64.whl
RUN pip install ./ktransformers-0.2.0+cu121torch23fancy-cp310-cp310-linux_x86_64.whl -i https://pypi.tuna.tsinghua.edu.cn/simple

ENTRYPOINT []

模型下载

# 创建目录
mkdir -p DeepSeek-R1-Q4_K_M
cd DeepSeek-R1-Q4_K_M

# 下载文件
for i in $(seq -f "%05g" 1 9); do
    wget -c https://www.modelscope.cn/models/unsloth/DeepSeek-R1-GGUF/resolve/master/DeepSeek-R1-Q4_K_M/DeepSeek-R1-Q4_K_M-$i-of-00009.gguf
done

ktransformers 单卡部署deepseek int4

version: '3.8'

services:
  ktransformers:
    image: registry.cn-shanghai.aliyuncs.com/zhph-server/ktransformers:0.2.0
    container_name: ktransformers
    ports:
      - 10002:10002
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    volumes:
      - /root/share_models/LLM/:/root/share_models/LLM/
    command:
      ktransformers --gguf_path /root/share_models/LLM/DeepSeek-R1-Q4_K_M --model_path /root/share_models/LLM/DeepSeek-R1 --cpu_infer 50 --port 10002 --web True --batch_size 1 --top_k 1 --modes True --paged True --model_name DeepSeek-R1 --model_dir /root/share_models/LLM/ --temperature 1 --cache_q4 True