Phase 4: training loop (code only — for user to run)
- src/train.py: AdamW (decay on matrices only), cosine LR w/ warmup, grad clipping, periodic train/val loss, CSV logging, resumable checkpoints. --overfit mode runs the overfit-one-batch sanity check. MPS, fp32. - configs/v1.py: TrainConfig (batch 64, peak LR 6e-4, 20k iters, etc.) - tests/test_train.py: LR schedule + optimizer grouping (3 tests, no run)
This commit is contained in:
+26
-1
@@ -21,5 +21,30 @@ class ModelConfig:
|
||||
d_ff: int = 1024 # MLP hidden width (4 x d_model)
|
||||
|
||||
|
||||
# The single source of truth other modules import.
|
||||
@dataclass(frozen=True)
|
||||
class TrainConfig:
|
||||
# Data / batching. tokens per step = batch_size * context_length = 16,384.
|
||||
batch_size: int = 64
|
||||
|
||||
# AdamW optimiser.
|
||||
learning_rate: float = 6e-4 # peak LR (reached after warmup)
|
||||
min_lr: float = 6e-5 # cosine-decay floor (~10% of peak)
|
||||
warmup_iters: int = 200
|
||||
max_iters: int = 20_000 # ~327M tokens (~0.6 epoch of the 555M corpus)
|
||||
weight_decay: float = 0.1 # applied to matrices only, not norms/biases
|
||||
beta1: float = 0.9
|
||||
beta2: float = 0.95
|
||||
grad_clip: float = 1.0 # clip global grad norm to this
|
||||
|
||||
# Evaluation, logging, checkpointing (all in optimiser steps).
|
||||
eval_interval: int = 500 # how often to measure train/val loss
|
||||
eval_iters: int = 100 # batches averaged per loss estimate
|
||||
log_interval: int = 20 # how often to print step/loss/lr/speed
|
||||
checkpoint_interval: int = 1000
|
||||
|
||||
seed: int = 1337
|
||||
|
||||
|
||||
# Single source of truth other modules import.
|
||||
model = ModelConfig()
|
||||
train = TrainConfig()
|
||||
|
||||
Reference in New Issue
Block a user