sglang

sglang--deepseek

环境

os：centos7.9

cuda：12.4

gcc：12.1

conda create -n SGLang python=3.10 -y

conda activate SGLan

pip install --upgrade pip

pip install torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1

pip install "sglang[all]" --find-links https://flashinfer.ai/whl/cu118/torch2.4/flashinfer/

python3 -m sglang.launch_server --model ./DeepSeek-R1-AWQ --tensor-parallel-size 8 --trust-remote-code --served-model-name deepseek-reasoner