Qwen3.5-27B-AWQ

pip install -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly

# Note: important for old containers, or could face run time error like
# subprocess.CalledProcessError: 
# Command '['ninja', '-v', '-C', 
# '/root/.cache/flashinfer/0.6.3/90a/cached_ops/trtllm_comm',
# '-f', '/root/.cache/flashinfer/0.6.3/90a/cached_ops/trtllm_comm/build.ninja']'
# returned non-zero exit status 1. 
rm -rf ~/.cache/flashinfer

# upgrade transformers so that applications could properly execute tool calls
pip install -U "transformers @ git+https://github.com/huggingface/transformers.git@f2ba019"
# locate modeling_rope_utils.py line 651 to fix a simple bug
TF_FILE="$(python -m pip show transformers | awk -F': ' '/^Location:/{print $2}')/transformers/modeling_rope_utils.py" && echo "$TF_FILE"
NEW_LINE='            ignore_keys_at_rope_validation = set(ignore_keys_at_rope_validation) | {"partial_rotary_factor"}' \
perl -i.bak -pe 'if ($. == 651) { $_ = $ENV{NEW_LINE} . "\n" }' "$TF_FILE"

uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=python&egg=sglang[all]'

uv pip install vllm --torch-backend=auto --extra-index-url https://wheels.vllm.ai/nightly

pip install "transformers[serving] @ git+https://github.com/huggingface/transformers.git@main"

transformers serve --force-model Qwen/Qwen3.5-27B --port 8000 --continuous-batching

pip install -U openai

# Set the following accordingly
export OPENAI_BASE_URL="http://localhost:8000/v1"
export OPENAI_API_KEY="EMPTY"