Qwen3-30B-A3B-Instruct-2507.w4a16

from llmcompressor.modifiers.quantization import GPTQModifier
  from llmcompressor.transformers import oneshot
  from transformers import AutoModelForCausalLM, AutoTokenizer
  
  # Load model
  model_stub = "Qwen/Qwen3-30B-A3B-Instruct"
  model_name = model_stub.split("/")[-1]

  num_samples = 1024
  max_seq_len = 8192

  model = AutoModelForCausalLM.from_pretrained(model_stub)

  tokenizer = AutoTokenizer.from_pretrained(model_stub)

  def preprocess_fn(example):
    return {"text": tokenizer.apply_chat_template(example["messages"], add_generation_prompt=False, tokenize=False)}
  
  ds = load_dataset("neuralmagic/LLM_compression_calibration", split="train")
  ds = ds.map(preprocess_fn)

  # Configure the quantization algorithm and scheme
  recipe = GPTQModifier(
      ignore: ["lm_head"]
      config_groups={"group_0": {"targets": ["Linear"], "weights": { "num_bits": 4, "type": int, "strategy": "group", "group_size": 128, "symmetric": true, "dynamic": false, "observer": "mse" } } },
      dampening_frac=0.01,
  )

  # Apply quantization
  oneshot(
      model=model,
      dataset=ds, 
      recipe=recipe,
      max_seq_length=max_seq_len,
      num_calibration_samples=num_samples,
  )
  
  # Save to disk in compressed-tensors format
  save_path = model_name + "-quantized.w4a16"
  model.save_pretrained(save_path)
  tokenizer.save_pretrained(save_path)
  print(f"Model and tokenizer saved to: {save_path}")