gemma-3-27b-it-INT4

# Server
export MODEL=pytorch/gemma-3-27b-it-INT4
VLLM_DISABLE_COMPILE_CACHE=1 vllm serve $MODEL --tokenizer $MODEL -O3

pip install git+https://github.com/huggingface/transformers@main
pip install torchao
pip install torch
pip install accelerate

pip install -U "huggingface_hub[cli]"
huggingface-cli login

export MODEL=google/gemma-3-27b-it # or pytorch/gemma-3-27b-it-INT4
lm_eval --model hf --model_args pretrained=$MODEL --tasks mmlu --device cuda:0 --batch_size 8

NUM_PROCESSES=8
MAIN_PORT=12345
MODEL_ID=google/gemma-3-27b-it # or pytorch/gemma-3-27b-it-INT4
TASKS=chartqa  # or tasks from https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/main/lmms_eval/models/simple
BATCH_SIZE=1
OUTPUT_PATH=./logs/

accelerate launch --num_processes "${NUM_PROCESSES}" --main_process_port "${MAIN_PORT}" -m lmms_eval \
  --model gemma3 \
  --model_args "pretrained=${MODEL_ID}" \
  --tasks "${TASKS}" \
  --batch_size "${BATCH_SIZE}" --output_path "${OUTPUT_PATH}"

git clone [email protected]:vllm-project/vllm.git

VLLM_USE_PRECOMPILED=1 pip install --editable .

export MODEL=google/gemma-3-27b-it
python benchmarks/benchmark_latency.py --input-len 256 --output-len 256 --model $MODEL --batch-size 1