contractor-ai-r1-70b-lora

# ContractorAI — DeepSeek-R1-Distill-Llama-70B QLoRA
# Hardware: 2×H100 SXM 80GB | DeepSpeed ZeRO-2

base_model: deepseek-ai/DeepSeek-R1-Distill-Llama-70B
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer
trust_remote_code: false

# ── Datasets ──────────────────────────────────────────────────────────────────
datasets:
  - path: /workspace/data/train.jsonl
    type: chat_template
    train_on_inputs: false
    message_property_mappings:
      role: role
      content: content

  - path: /workspace/data/advanced/chain_of_thought.jsonl
    type: chat_template
    train_on_inputs: false
    message_property_mappings:
      role: role
      content: content
    ds_weight: 2.0

  - path: /workspace/data/advanced/multi_turn.jsonl
    type: chat_template
    train_on_inputs: false
    message_property_mappings:
      role: role
      content: content
    ds_weight: 2.0

  - path: /workspace/data/advanced/reasoning_2026.jsonl
    type: chat_template
    train_on_inputs: false
    message_property_mappings:
      role: role
      content: content
    ds_weight: 2.0

  - path: /workspace/data/advanced/tool_use.jsonl
    type: chat_template
    train_on_inputs: false
    message_property_mappings:
      role: role
      content: content
    ds_weight: 1.5

dataset_prepared_path: /workspace/prepared
val_set_size: 0.05
output_dir: /bstone777/contractor-ai-r1-70b-lora

# ── Precision ─────────────────────────────────────────────────────────────────
load_in_4bit: true
bnb_4bit_use_double_quant: true
bnb_4bit_quant_type: nf4
torch_dtype: bfloat16
bf16: true
fp16: false

# ── QLoRA ─────────────────────────────────────────────────────────────────────
adapter: lora
lora_r: 32
lora_alpha: 64
lora_dropout: 0.05
lora_target_linear: true

# ── Training ──────────────────────────────────────────────────────────────────
sequence_len: 2048
sample_packing: true
pad_to_sequence_len: false

num_epochs: 3
micro_batch_size: 1
gradient_accumulation_steps: 16
learning_rate: 2e-4
lr_scheduler: cosine
warmup_steps: 100
weight_decay: 0.01
max_grad_norm: 1.0
optimizer: adamw_bnb_8bit

# ── Attention ─────────────────────────────────────────────────────────────────
sdp_attention: true           # PyTorch SDPA — built into torch 2.8
flash_attention: false

# ── Gradient Checkpointing ────────────────────────────────────────────────────
gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: true

# ── Eval & Checkpointing ──────────────────────────────────────────────────────
eval_steps: 100
eval_strategy: steps
save_steps: 25
save_total_limit: 3
logging_steps: 10
load_best_model_at_end: true
metric_for_best_model: eval_loss

# ── Chat Template ─────────────────────────────────────────────────────────────
chat_template: llama3
default_system_message: "You are ContractorAI, an expert AI assistant for the construction and home improvement industry with deep reasoning capabilities. You excel at cost estimating, building codes, sales, and coordinating multi-agent workflows for contracting businesses."

special_tokens:
  pad_token: "<|finetune_right_pad_id|>"

# ── DeepSpeed ─────────────────────────────────────────────────────────────────
deepspeed: /workspace/configs/deepspeed_zero3.json

# ── Misc ─────────────────────────────────────────────────────────────────────
seed: 42
strict: false