Red-Pilled-Morpheus-7B-v1

MAX_SEQ_LENGTH = 1024  # Increased to capture full context
LORA_R = 8            # Increased for better "intelligence" retention # 8 / 16 barely fits on a 3060 ti with 250Q&A. Increasing to 5K Q&A requires reducing to 4 / 8.
LORA_ALPHA = 16 # Increase these after GPU upgrade (Lora_R * 2 = Lora_Alpha)
NUM_EPOCHS = 5
LEARNING_RATE = 1e-4
optim="paged_adamw_8bit",
max_grad_norm=0.3,
warmup_ratio=0.03,
lr_scheduler_type="cosine",
lora_dropout=0.05,
target_modules=[
            "q_proj", "k_proj", "v_proj", "o_proj",
            "gate_proj", "up_proj", "down_proj", # up_proj and down_proj may have to be commented out for bigger datasets due to VRAM limits
# --- SAVE STRATEGY PATCH ---
        ### save_strategy="steps",        # Set to steps for large datasets
        ### save_steps=100,                # Save every 100 steps
        save_strategy="epoch",    # ### HOTSWAP: Uncomment for small datasets