Add degradation experiments: one-parameter-at-a-time ablations
Controlled study of how the model degrades. Four variants, each = v1 with a single field replaced (dataclasses.replace), so differences are attributable: - small: capacity down (2L/64d, ~0.38M params) - short: training time down (max_iters 20k->2k) - ctx: context window down (256->64) - data: data quantity down (max_train_tokens->1M; new TrainConfig knob + train.py slice) - scripts/compare.py: sample same prompt across all trained configs with a shared seed, print side by side, write reports/compare.md - tests/test_configs.py: enforces one-parameter-at-a-time (only intended fields differ from v1) + small param count (3 tests). Full suite: 29 passing.
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
"""Compare generated text across the degradation experiments.
|
||||
|
||||
Samples the same prompt(s) from every config that has a trained checkpoint,
|
||||
using the same RNG seed for each so differences reflect the model, not sampling
|
||||
luck. Prints the results side by side and writes reports/compare.md.
|
||||
|
||||
Usage:
|
||||
python scripts/compare.py # default prompt, all configs
|
||||
python scripts/compare.py --prompt "The dog and the cat"
|
||||
python scripts/compare.py --suite # every suite prompt
|
||||
python scripts/compare.py --configs v1,small,ctx # a subset
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
from sample import ( # noqa: E402
|
||||
SUITE_PROMPTS,
|
||||
TOK_PATH,
|
||||
load_model,
|
||||
pick_device,
|
||||
sample_text,
|
||||
)
|
||||
from tokenizer import BPETokenizer # noqa: E402
|
||||
|
||||
# Baseline first, then the four one-parameter degradations.
|
||||
DEFAULT_CONFIGS = ["v1", "small", "short", "ctx", "data"]
|
||||
|
||||
LABELS = {
|
||||
"v1": "v1 (baseline)",
|
||||
"small": "small (less capacity)",
|
||||
"short": "short (less training)",
|
||||
"ctx": "ctx (smaller context)",
|
||||
"data": "data (less data)",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--configs", default=",".join(DEFAULT_CONFIGS))
|
||||
ap.add_argument("--prompt", default="Once upon a time,")
|
||||
ap.add_argument("--suite", action="store_true")
|
||||
ap.add_argument("--max-new-tokens", type=int, default=200)
|
||||
ap.add_argument("--temperature", type=float, default=0.8)
|
||||
ap.add_argument("--top-k", type=int, default=200)
|
||||
ap.add_argument("--seed", type=int, default=1234)
|
||||
args = ap.parse_args()
|
||||
|
||||
configs = [c.strip() for c in args.configs.split(",") if c.strip()]
|
||||
prompts = SUITE_PROMPTS if args.suite else [args.prompt]
|
||||
|
||||
device = pick_device()
|
||||
tok = BPETokenizer.load(TOK_PATH)
|
||||
|
||||
# Load each available model once; warn about any not yet trained.
|
||||
models = {}
|
||||
for name in configs:
|
||||
if (ROOT / "checkpoints" / name / "ckpt.pt").exists():
|
||||
models[name] = load_model(name, device)
|
||||
else:
|
||||
print(f"[skip {name}: no checkpoint — train it first]")
|
||||
if not models:
|
||||
sys.exit("no trained checkpoints found among: " + ", ".join(configs))
|
||||
|
||||
out = ["# Degradation comparison", ""]
|
||||
for prompt in prompts:
|
||||
print(f"\n########## prompt: {prompt!r} ##########")
|
||||
out += [f"## Prompt: {prompt}", ""]
|
||||
for name, model in models.items():
|
||||
torch.manual_seed(args.seed) # same RNG state per model = fair compare
|
||||
text = sample_text(model, tok, prompt, device,
|
||||
args.max_new_tokens, args.temperature, args.top_k)
|
||||
label = LABELS.get(name, name)
|
||||
print(f"\n----- {label} -----\n{text}")
|
||||
out += [f"### {label}", "", f"> {text}", ""]
|
||||
|
||||
out_path = ROOT / "reports" / "compare.md"
|
||||
out_path.parent.mkdir(exist_ok=True)
|
||||
out_path.write_text("\n".join(out))
|
||||
print(f"\nsaved {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user