Phase 1: trained 4096-vocab tokenizer + per-word encode cache

- artifacts/tokenizer.json: 3839 merges, trained on 50MB sample in 87s.
  3.97 bytes/token on held-out text.
- encode() caches word->ids so encoding the full corpus is mostly dict
  lookups (TinyStories has a small vocabulary).

src/tokenizer.py

@@ -63,6 +63,9 @@ class BPETokenizer:
         # rank = merge priority (lower merges first); id of a merged pair = 256 + rank
         self.ranks = {pair: rank for rank, pair in enumerate(merges)}
         self.vocab = self._build_vocab(merges, special_tokens)
+        # Cache word -> ids. TinyStories has a small vocabulary, so encoding the
+        # whole corpus becomes mostly dict lookups instead of repeated merging.
+        self._chunk_cache: dict[str, list[int]] = {}
 
     @staticmethod
     def _build_vocab(
@@ -164,7 +167,11 @@ class BPETokenizer:
                 ids.append(self.special_tokens[segment])
             else:
                 for chunk in pre_tokenize(segment):
-                    ids.extend(self._encode_chunk(chunk))
+                    cached = self._chunk_cache.get(chunk)
+                    if cached is None:
+                        cached = self._encode_chunk(chunk)
+                        self._chunk_cache[chunk] = cached
+                    ids.extend(cached)
         return ids
 
     def decode(self, ids: list[int]) -> str: