seed-coder-triton-8b-v1

base_model: ByteDance-Seed/Seed-Coder-8B-Base

plugins:
  - axolotl.integrations.liger.LigerPlugin
  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
liger_rope: true
liger_rms_norm: true
liger_glu_activation: true

chat_template: llama3
datasets:
  - path: axolotl-ai-internal/gpumode-py2triton-reasoning-v2
    type: chat_template
    split: train

dataset_prepared_path: last_run_prepared
val_set_size: 0.005
output_dir: ./outputs/out

sequence_len: 16384
sample_packing: true
pad_to_sequence_len: true

wandb_project: seed-coder-8b-grpo-triton
wandb_entity: axolotl-ai
wandb_watch:
wandb_name:
wandb_log_model:

gradient_accumulation_steps: 1
micro_batch_size: 2
num_epochs: 3
optimizer: adamw_torch_fused
max_grad_norm: 0.1
neftune_noise_alpha: 10
lr_scheduler: cosine
learning_rate: 1e-6
lr_groups:
  - name: embeddings
    modules:
      - embed_tokens
      - lm_head
    lr: 0.00003  # scalu up LR for embeddings as these are unused tokens

bf16: true
tf32: true

gradient_checkpointing: offload
gradient_checkpointing_kwargs:
  use_reentrant: false
logging_steps: 1
flash_attention: true

warmup_steps: 100
evals_per_epoch: 5
saves_per_epoch: 1
weight_decay: 0.01
deepspeed: deepspeed_configs/zero1.json
special_tokens:
  eos_token: <|eot_id|>
added_tokens_overrides:
  7: <|start_header_id|>
  8: <|end_header_id|>
  9: <|eot_id|>
  10: <think>
  11: </think>
fix_untrained_tokens: [7, 8, 9, 10, 11]