Llama-Guard-4-12B-FP8-dynamic

CUDA_VISIBLE_DEVICES=0 python quantize.py --model_path meta-llama/Llama-Guard-4-12B RedHatAI/Llama-Guard-4-12B-FP8-dynamic --pipeline datafree

import argparse
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, Llama4ForConditionalGeneration
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor import oneshot
from compressed_tensors.quantization import (
    QuantizationScheme,
    QuantizationArgs,
    QuantizationType,
    QuantizationStrategy,
)


def main():
    parser = argparse.ArgumentParser(description="Quantize a causal language model")
    parser.add_argument(
        "--model_path",
        type=str,
        required=True,
        help="Path to the pre-trained model",
    )
    parser.add_argument(
        "--quant_path",
        type=str,
        required=True,
        help="Output path for the quantized model",
    )
    parser.add_argument(
        "--pipeline", #['basic', 'datafree', 'sequential', independent]
        type=str,
        required=True,
    )

    print(f"Loading model from {args.model_path}...")
    model = Llama4ForConditionalGeneration.from_pretrained(
        args.model_path,
        torch_dtype="auto",
        trust_remote_code=True,
    )

    recipe = QuantizationModifier(
        targets="Linear",
        scheme="FP8_dynamic",
        ignore=[
            're:.*lm_head',
            're:.*multi_modal_projector',
            're:.*vision_model',
        ]
    )

    print("Applying quantization...")
    oneshot(
        model=model,
        recipe=recipe,
        trust_remote_code_model=True,
        pipeline=args.pipeline,
    )

    model.save_pretrained(args.quant_path, save_compressed=True, skip_compression_stats=True, disable_sparse_compression=True)
    print(f"Quantized model saved to {args.quant_path}")


if __name__ == "__main__":
    main()