speculative-cascades

from transformers import AutoModelForCausalLM, AutoTokenizer
model = AutoModelForCausalLM.from_pretrained('Qwen/Qwen3-1.7B')
assistant_model = AutoModelForCausalLM.from_pretrained('Qwen/Qwen3-0.6B')

tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen3-1.7B')

question = "If I bought a shirt at 20% discount for $4, what was the initial price before the discount?"
inputs = tokenizer(question, return_tensors='pt')

output = model.generate(
    **inputs,
    assistant_model=assistant_model,
    do_sample=True,
    alpha=0.25, # Rate at which to defer to the target model
    deferral='v3', # Deferral method. See paper for details.
    custom_generate='radia/speculative-cascades',
    trust_remote_code=True,
    max_new_tokens=320,
)

outputs = model.generate(
    assistant_model
)

print(tokenizer.decode(outputs.tolist()[0]))