Nemotron2Gemma-AURORA-LoRA-27B-IT-0p95

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel

base_id = "Changgil/google-gemma-3-27b-it-text"
adapter_id = "win10/Nemotron2Gemma-AURORA-LoRA-27B-IT-0p95"

tokenizer = AutoTokenizer.from_pretrained(base_id, use_fast=True)

base = AutoModelForCausalLM.from_pretrained(
    base_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
)

model = PeftModel.from_pretrained(base, adapter_id)
model.eval()

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Explain knowledge distillation in 5 bullet points."},
]

inputs = tokenizer.apply_chat_template(
    messages,
    add_generation_prompt=True,
    tokenize=True,
    return_tensors="pt",
)

with torch.no_grad():
    out = model.generate(
        inputs.to(model.device),
        max_new_tokens=512,
        do_sample=False,
    )

print(tokenizer.decode(out[0], skip_special_tokens=True))

from peft import PeftModel

merged = model.merge_and_unload()
merged.save_pretrained("./merged_model", safe_serialization=True)
tokenizer.save_pretrained("./merged_model")

python universal_distill_v4_1_0_aurora_svd_innovations.py \
  --teacher E:\text-generation-webui-1.14\user_data\models\Llama-3.1-Nemotron-70B-Instruct-HF \
  --student E:\text-generation-webui-1.14\user_data\models\google-gemma-3-27b-it-text \
  --output  ./Llama-3.1-Nemotron-70B-Instruct-HF-gemma-3-27b-it-text-lora-adaptive \
  --svd-mode aurora \
  --energy-threshold 0.95 \
  --min-rank 256 \
  --max-rank 5376 \
  --interp-mode lsq \
  --svd-rand-iter 2 \
  --svd-rand-oversamples 8 \
  --svd-aurora-steps 100 \
  --svd-aurora-order 2 \
  --calib-format alpaca \
  --calib-alpaca-template classic \
  --calib-max-samples 128 \
  --calib-max-length 65536 \
  --calib-batch-size 2 \
  --calib-save .\calib_stats_Yi-70B-200k_alpaca-taiwan-dataset.safetensors \
  --calib-mode rms \
  --include "self_attn|mlp"