attn_impl: sdpa
base_model: meta-llama/Llama-3.1-8B-Instruct
batch_size: 8
compute_dtype: bfloat16
epochs: 3
grad_accum: 1
learning_rate: 0.0002
logging_steps: 10
lora_alpha: 32
lora_dropout: 0.05
lora_r: 16
lora_target_modules:
- q_proj
- k_proj
- v_proj
- o_proj
- gate_proj
- up_proj
- down_proj
lr_scheduler: cosine
max_seq_length: 3072
output_dir: /home/shadeform/wireclaw-training/output/wireclaw-v1.3.1-brev
seed: 42
train_file: /home/shadeform/wireclaw-training-data/v1.3.1-train.jsonl
val_file: /home/shadeform/wireclaw-training-data/wireclaw-v2-val.jsonl
warmup_ratio: 0.03
weight_decay: 0.01