Phase 4: training loop (code only — for user to run)

- src/train.py: AdamW (decay on matrices only), cosine LR w/ warmup, grad
  clipping, periodic train/val loss, CSV logging, resumable checkpoints.
  --overfit mode runs the overfit-one-batch sanity check. MPS, fp32.
- configs/v1.py: TrainConfig (batch 64, peak LR 6e-4, 20k iters, etc.)
- tests/test_train.py: LR schedule + optimizer grouping (3 tests, no run)
This commit is contained in:
2026-07-12 10:51:58 -04:00
parent 7b25fc89f6
commit 234a37f8fd
3 changed files with 299 additions and 1 deletions
+26 -1
View File
@@ -21,5 +21,30 @@ class ModelConfig:
d_ff: int = 1024 # MLP hidden width (4 x d_model)
# The single source of truth other modules import.
@dataclass(frozen=True)
class TrainConfig:
# Data / batching. tokens per step = batch_size * context_length = 16,384.
batch_size: int = 64
# AdamW optimiser.
learning_rate: float = 6e-4 # peak LR (reached after warmup)
min_lr: float = 6e-5 # cosine-decay floor (~10% of peak)
warmup_iters: int = 200
max_iters: int = 20_000 # ~327M tokens (~0.6 epoch of the 555M corpus)
weight_decay: float = 0.1 # applied to matrices only, not norms/biases
beta1: float = 0.9
beta2: float = 0.95
grad_clip: float = 1.0 # clip global grad norm to this
# Evaluation, logging, checkpointing (all in optimiser steps).
eval_interval: int = 500 # how often to measure train/val loss
eval_iters: int = 100 # batches averaged per loss estimate
log_interval: int = 20 # how often to print step/loss/lr/speed
checkpoint_interval: int = 1000
seed: int = 1337
# Single source of truth other modules import.
model = ModelConfig()
train = TrainConfig()