Qwen3-235B-A22B-NVFP4

290

8 languages

license:apache-2.0

by

RedHatAI

Language Model

OTHER

235B params

New

290 downloads

Early-stage

Try on Hugging Face Add to Compare

Edge AI:

Mobile

Laptop

Server

526GB+ RAM

Mobile

Laptop

Server

Quick Summary

Model Overview - Model Architecture: Qwen/Qwen3-235B-A22B - Input: Text - Output: Text - Model Optimizations: - Weight quantization: FP4 - Activation quantizati...

Device Compatibility

Mobile

4-6GB RAM

Laptop

16GB RAM

Server

GPU

Minimum Recommended

219GB+ RAM

Code Examples

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

--- Replace MoE modules for calibration ---pythontransformers

from datasets import load_dataset

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.modifiers.quantization import GPTQModifier
from llmcompressor.modifiers.smoothquant import SmoothQuantModifier

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from llmcompressor.modeling.prepare import replace_modules_for_calibration

MODEL_ID = "Qwen/Qwen3-235B-A22B"

 #Load model.
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map=None, torch_dtype="auto"
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print(model)


tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

DATASET_ID = "HuggingFaceH4/ultrachat_200k"
DATASET_SPLIT = "train_sft"

NUM_CALIBRATION_SAMPLES = 256
MAX_SEQUENCE_LENGTH = 1024

# --- Replace MoE modules for calibration ---
model = replace_modules_for_calibration(model, calibrate_all_experts=False)

# Load dataset and preprocess.
ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
ds = ds.shuffle(seed=42)

def preprocess(example):
    return {
        "text": tokenizer.apply_chat_template(
            example["messages"],
            tokenize=False,
        )
    }


ds = ds.map(preprocess)

# Tokenize inputs.
def tokenize(sample):
    return tokenizer(
        sample["text"],
        padding=False,
        max_length=MAX_SEQUENCE_LENGTH,
        truncation=True,
        add_special_tokens=False,
    )


ds = ds.map(tokenize, remove_columns=ds.column_names)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["re:.*lm_head.*", "re:.*mlp.gate$", "re:.*self_attn",
],
)

# Save to disk in compressed-tensors format.
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"

# Apply quantization.
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
    output_dir=SAVE_DIR,
    pipeline="sequential",
    sequential_targets=["Qwen3MoeDecoderLayer"],
    calibrate_moe_context=True,

)
# Save to disk in compressed-tensors format.
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)

Deploy This Model

Production-ready deployment in minutes

Together.ai

Instant API access to this model

Fastest API

Production-ready inference API. Start free, scale to millions.

Replicate

One-click model deployment

Easiest Setup

Run models in the cloud with simple API. No DevOps required.

Disclosure: We may earn a commission from these partners. This helps keep LLMYourWay free.