diff --git a/marker_experiments/.gitignore b/marker_experiments/.gitignore new file mode 100644 index 00000000..d1431a3d --- /dev/null +++ b/marker_experiments/.gitignore @@ -0,0 +1,6 @@ +# Rebuildable and far too large to track: pretokenized corpora (~90MB each x24), +# on-disk text blocks (~1GB per language), held-out eval slices, BPE init caches. +corpora/ +# (text blocks no longer written: streamed straight from parquet) +eval_texts/ +bpe_init_cache/ diff --git a/marker_experiments/boundary_pretokenizer.py b/marker_experiments/boundary_pretokenizer.py new file mode 100644 index 00000000..8875085a --- /dev/null +++ b/marker_experiments/boundary_pretokenizer.py @@ -0,0 +1,396 @@ +"""Boundary-marker pretokenization for SCRIPT encoding. + +A single atomic token `<|>` delimits *spans*. The single space between two +adjacent delimited spans is elided at encode time and reconstructed at decode +time from the resulting pair of touching markers: + + two <|> touching -> exactly one space + a lone <|> -> nothing (structural boundary only) + +This removes the with/without-space duplication that the leading-space +convention creates (`' the'` and `'the'` as separate vocabulary entries). + +Span identification +------------------- +A **word span** is a maximal run of characters from *any* space-using script +(`DEFAULT_SCRIPTS_LM_WITH_SPACES`, category LM), merged across script changes. +So `latin` immediately followed by Cyrillic `кириллица` is ONE span, delimited +only at its outer edges: + + <|>latin кириллица<|> (no marker between them) + +Merging across scripts is what makes the scheme well defined. If each script run +were delimited separately, two unconditional markers would meet at the script +change, they would be indistinguishable from an elided space, and decode would +fabricate one. Merging first means two word spans can never be adjacent, so the +touching-marker signal is unambiguous with no special case. + +Inside a span the baseline's script-based chunk split is preserved (the marker +rides on the first and last chunk), so no BPE merge ever spans a script change +that the baseline would not also allow. + +Boundary targets +---------------- +`boundary_targets` selects which unit kinds are delimited: + + ("word",) - word spans only + ("word", "punct") - and punctuation + ("word", "punct", "digit") - and digits + +Word spans are delimited on BOTH sides unconditionally, which is the point: a +word has one canonical form regardless of what precedes it. Punctuation and +digits are delimited only on a side whose adjacent single space was elided. +That asymmetry is required: marking punctuation unconditionally would make `a,b` +encode as `<|>a<|> <|>,<|> <|>b<|>`, with touching markers at both junctions and +therefore indistinguishable from `a , b`. Marking only on a space side keeps the +invariant, at the cost of up to four variants per mark (`,` `,<|>` `<|>,` +`<|>,<|>`) -- affordable because punctuation and digits are closed sets, unlike +words. + +Digits are a separate target because `script_category_v3` folds L/M into LM, +Z/Cc into ZC, So into So and P/S/Cf into PSF, but leaves category N alone, so +digits are neither letters nor `combines_with_spaces`. +""" + +import itertools +from typing import Literal, Sequence + +from pydantic import ConfigDict + +from script_bpe.pretokenize.pretokenizer import ( + CharEncT, + ScriptPretokenizer, + ScriptPretokenizerConfig, + group_digits, +) + +BoundaryTarget = Literal["word", "punct", "digit"] + + +class MarkerCharEnc: + """Stand-in char encoding for the marker token, inserted directly rather than scanned.""" + + __slots__ = ("script_id", "combines_with_spaces", "atomic_token_ids", "inherited") + + def __init__(self, token_id: int): + self.script_id = -2 # sentinel: never produced by groupby over real text + self.combines_with_spaces = False + self.inherited = False + self.atomic_token_ids = [token_id] + + def __repr__(self): + return f"MarkerCharEnc(atomic_token_ids={self.atomic_token_ids})" + + +class CodeCharEnc(MarkerCharEnc): + """Stand-in char encoding for a caps code. Distinct sentinel so it never groups with + real text or with the boundary marker.""" + + def __init__(self, token_id: int): + super().__init__(token_id) + self.script_id = -3 + + +class BoundaryScriptPretokenizerConfig(ScriptPretokenizerConfig): + cls: str = "BoundaryScriptPretokenizer" + boundary_targets: tuple[BoundaryTarget, ...] = ("word", "punct", "digit") + # Caps codes, in the style of the older Claude tokenizer: a title-case word is emitted + # as a shift code plus its lowercased form, an all-caps word as a caps-lock code plus + # its lowercased form, so 'The'/'the' and 'NASA'/'nasa' share vocabulary entries. Whole + # spans only; mixed case ('GaN', 'WiFi') is left literal. + caps_codes: bool = False + model_config = ConfigDict(extra="forbid") + + +class BoundaryScriptPretokenizer(ScriptPretokenizer, config_type=BoundaryScriptPretokenizerConfig): + MARKER_TEXT = "<|>" + SHIFT_TEXT = "<^>" # title case: next span is Xxxx + CAPS_TEXT = "<^^>" # caps lock: next span is XXXX + + # unit kinds + WORD, PUNCT, DIGIT, SPACE, OTHER = "word", "punct", "digit", "space", "other" + + def _build_atomic_tokens(self): + super()._build_atomic_tokens() + self.marker_token_id = self._register_token(self.MARKER_TEXT) + self.is_initial_char_tokens.add(self.marker_token_id) + if self.config.caps_codes: + self.shift_token_id = self._register_token(self.SHIFT_TEXT) + self.caps_token_id = self._register_token(self.CAPS_TEXT) + self.is_initial_char_tokens.add(self.shift_token_id) + self.is_initial_char_tokens.add(self.caps_token_id) + else: + self.shift_token_id = self.caps_token_id = None + blocks = self.config.script_config.blocks + # the ~20 space-using writing systems, as letters + self.word_script_ids = {b.script_id for b in blocks if b.category == "LM" and b.combines_with_spaces} + self.digit_script_ids = {b.script_id for b in blocks if b.category == "N"} + targets = set(self.config.boundary_targets) + unknown = targets - {"word", "punct", "digit"} + if unknown: + raise ValueError(f"Unknown boundary_targets: {sorted(unknown)}") + # kinds that carry a boundary and may therefore participate in space elision + self.marked_kinds = frozenset( + k for k, name in ((self.WORD, "word"), (self.PUNCT, "punct"), (self.DIGIT, "digit")) + if name in targets + ) + + def __init__(self, config: BoundaryScriptPretokenizerConfig) -> None: + super().__init__(config) + # Digit-group tokens are registered by the base _build_digit_tokens AFTER + # _build_atomic_tokens runs, and ScriptPretokenizer.decode has no path for them + # (digit_handling was only ever exercised with UTF8Pretokenizer). Collect their + # ids here so decode can emit them directly; they are the only atomic tokens + # whose text is all digits. + self.digit_token_ids = {tid for tid, txt in self.atomic_tokens.items() if txt.isdigit()} + + def bpe_merge_allowed(self, a, b) -> bool: + # No learned token may span an elided-space point. Without this, BPE learns tokens + # like '<|>the<|><|>' that swallow the dangling half of the next span's opening + # marker, reintroducing per-word duplication keyed on what follows. + if a[-1] == self.marker_token_id and b[0] == self.marker_token_id: + return False + return super().bpe_merge_allowed(a, b) + + def decode(self, tokenization, errors="replace") -> str: + decoded = "" + pending = None # caps code awaiting its span + buf = "" + i = 0 + n = len(tokenization) + + def emit(text): + nonlocal decoded, pending, buf + if pending is None: + decoded += text + else: + buf += text + + def flush(): + """Close a caps-coded span at its terminating marker.""" + nonlocal decoded, pending, buf + if pending is None: + return + decoded += (buf[:1].upper() + buf[1:]) if pending == "shift" else buf.upper() + pending, buf = None, "" + + while i < n: + if self.config.caps_codes and tokenization[i] in (self.shift_token_id, self.caps_token_id): + flush() # a code immediately after another closes the previous span + pending = "shift" if tokenization[i] == self.shift_token_id else "caps" + buf = "" + i += 1 + continue + if tokenization[i] == self.marker_token_id: + flush() + if i + 1 < n and tokenization[i + 1] == self.marker_token_id: + decoded += " " # two markers touching == one elided space + i += 2 + else: + i += 1 # lone marker: structural boundary, no character + continue + if tokenization[i] in self.digit_token_ids: + emit(self.atomic_tokens[tokenization[i]]) # digit group, single token + i += 1 + continue + script_tok = tokenization[i] + ix_tok = tokenization[i + 1] if i + 1 < n else None + if (script_tok, ix_tok) in self.detokenize_map: + emit(self.detokenize_map[(script_tok, ix_tok)]) + i += 2 + else: + if errors == "backslashreplace": + emit(self.atomic_tokens[script_tok]) + elif errors == "replace": + decoded += "�" + elif errors == "strict": + raise ValueError(f"Invalid tokenization: ({script_tok}, {ix_tok}) is not a valid token pair!") + else: + raise ValueError(f"Unknown error handling mode: {errors}") + i += 1 + flush() # span running to end of stream + return decoded + + @staticmethod + def _caps_form(text: str): + """Return (code_kind, lowercased) if text is title or all caps AND the transform is + exactly invertible, else None. + + Invertibility cannot be assumed. Unicode case mapping is not a bijection: 'I'.lower() + is 'i' but Turkish dotless/dotted i break the pair, '\u0130'.lower() is two + characters, and '\u1e9e'.lower() is '\u00df' whose upper is 'SS'. Every candidate is + therefore verified by re-applying the transform and comparing, and anything that does + not reproduce the source exactly is left literal. + """ + if not text or text.islower(): + return None + low = text.lower() + if len(low) != len(text): + return None + if low[0].upper() + low[1:] == text: + return "shift", low + if len(text) > 1 and low.upper() == text: + return "caps", low + return None + + def _kind(self, group) -> str: + script_id = group[0][1].script_id + if script_id in self.word_script_ids: + return self.WORD + if script_id in self.digit_script_ids: + return self.DIGIT + if group[0][1].combines_with_spaces: + return self.PUNCT + return self.OTHER + + def _build_units(self, script_groups) -> list[tuple[str, list[list]]]: + """Group script runs into units. A unit holds a LIST of script runs, so a word span + that crosses scripts keeps its internal split while being one delimited span.""" + units: list[tuple[str, list[list]]] = [] + i = 0 + n = len(script_groups) + while i < n: + group = script_groups[i] + if [e for _, e in group] == self.space_group: # exactly one space character + units.append((self.SPACE, [list(group)])) + i += 1 + continue + kind = self._kind(group) + runs = [list(group)] + i += 1 + if kind == self.WORD: + # merge across ANY space-using-script change, plus inherited marks + while i < n and ( + script_groups[i][0][1].inherited or script_groups[i][0][1].script_id in self.word_script_ids + ): + if ( + script_groups[i][0][1].inherited + or script_groups[i][0][1].script_id == runs[-1][0][1].script_id + ): + runs[-1] = runs[-1] + list(script_groups[i]) + else: + runs.append(list(script_groups[i])) + i += 1 + else: + script_id = group[0][1].script_id + while i < n and ( + script_groups[i][0][1].inherited or script_groups[i][0][1].script_id == script_id + ): + runs[-1] = runs[-1] + list(script_groups[i]) + i += 1 + units.append((kind, runs)) + return units + + def split_unencoded_and_encode(self, text: str) -> list[Sequence[CharEncT]]: + """Encode and chunk in one pass, keeping source characters alongside encodings. + + The base implementation splits digit runs into their own chunks *before* + split_encoded runs. That is fatal here: a digit unit and its neighbouring word + would land in different chunks, so the shared single space between them could + never be seen as elidable, and `digit` as a boundary target would silently do + nothing. The unit analysis therefore has to happen over the whole text first, + with digit grouping applied inside a digit unit afterwards. + """ + if self.config.regex_pattern is not None: + raise NotImplementedError("BoundaryScriptPretokenizer does not support regex_pattern") + enc = self.encode_text(text) # 1:1 with characters + pairs = list(zip(text, enc)) + groups = [list(g) for _, g in itertools.groupby(pairs, key=lambda p: p[1].script_id)] + units = self._build_units(groups) + marker = MarkerCharEnc(self.marker_token_id) + marked = self.marked_kinds + + # A single space is elided when BOTH neighbours carry a boundary; their facing + # sides then hold markers, which land adjacent in the atomic stream. + elided = [False] * len(units) + for i, (kind, _) in enumerate(units): + if kind != self.SPACE: + continue + if 0 < i < len(units) - 1 and units[i - 1][0] in marked and units[i + 1][0] in marked: + elided[i] = True + + chunks: list[Sequence[CharEncT]] = [] + for i, (kind, runs) in enumerate(units): + if kind == self.SPACE: + if not elided[i]: + chunks.append([e for _, e in runs[0]]) # exactly as the baseline emits it + continue + digits = "".join(c for run in runs for c, _ in run) if kind == self.DIGIT else "" + # group_digits/encode_digits only have tokens for ASCII 0-9 -- the base pipeline + # splits on re.split("([0-9]+)"). Category N is far broader (Nd for every script, + # plus Nl/No: '½', '⅓', '٣', 'Ⅻ'), so grouping those raises KeyError. They stay on + # the ordinary script path; they are still delimited, but their marked forms are + # not bounded by grouping. + if kind == self.DIGIT and self.config.digit_handling is not None and digits.isascii() and digits.isdigit(): + # Split the digit run into groups so marked forms stay bounded: with a + # whole run as one unit, every distinct number acquires up to four marked + # variants, which measured 1,093 wasted vocabulary slots (3.17%) for + # English at 32,768. Splitting means only the run's first and last GROUP + # can carry a marker -- 10 digits under SPLIT, 1110 under RTL3. + digits = "".join(c for run in runs for c, _ in run) + out = [self.encode_digits([g]) for g in group_digits(digits, self.config.digit_handling)] + else: + out = [[e for _, e in run] for run in runs] + if kind == self.WORD and self.config.caps_codes: + text = "".join(c for run in runs for c, _ in run) + form = self._caps_form(text) + if form is not None: + code, low = form + # Re-encode the lowercased span and regroup, so a span that crosses + # scripts keeps the same internal split it would have had untouched. + low_pairs = list(zip(low, self.encode_text(low))) + low_runs = [list(g) for _, g in itertools.groupby(low_pairs, key=lambda x: x[1].script_id)] + out = [[e for _, e in r] for r in low_runs] + code_id = self.shift_token_id if code == "shift" else self.caps_token_id + out[0] = [CodeCharEnc(code_id)] + out[0] + if kind not in marked: + chunks.extend(out) + continue + if kind == self.WORD: + left = right = True # unconditional: one canonical form per span + else: + left = i > 0 and elided[i - 1] + right = i + 1 < len(units) and elided[i + 1] + # marker rides the first/last run, preserving any internal split + if left: + out[0] = [marker] + list(out[0]) + if right: + out[-1] = list(out[-1]) + [marker] + chunks.extend(out) + return chunks + + def split_encoded(self, encoding: Sequence[CharEncT]) -> list[Sequence[CharEncT]]: + # All chunking already happened in split_unencoded_and_encode. + return [encoding] + + +# Named variants used by the experiments. All are ScriptEncodingV3 with +# enforce_char_boundaries=True; they differ only in which units get a boundary. +BOUNDARY_VARIANTS = { + "bnd_w": ("word",), + "bnd_wp": ("word", "punct"), + "bnd_wpd": ("word", "punct", "digit"), +} + + +# Caps-code variants: same boundary targets, plus <^>/<^^> case codes on word spans. +# Kept in a separate table because the grids iterate BOUNDARY_VARIANTS to enumerate cells. +CAPS_VARIANTS = {f"{name}_caps": targets for name, targets in BOUNDARY_VARIANTS.items()} + +ALL_VARIANTS = {**BOUNDARY_VARIANTS, **CAPS_VARIANTS} + + +def get_boundary_pretokenizer(name: str, **overrides) -> BoundaryScriptPretokenizer: + """Build a named variant. `overrides` are passed to the config (e.g. digit_handling).""" + from script_bpe.pretokenize.scriptencoding import ScriptEncodingV3 + + if name not in ALL_VARIANTS: + raise ValueError(f"unknown boundary variant {name!r}; have {sorted(ALL_VARIANTS)}") + return BoundaryScriptPretokenizer( + BoundaryScriptPretokenizerConfig( + script_config=ScriptEncodingV3, + boundary_targets=ALL_VARIANTS[name], + caps_codes=name in CAPS_VARIANTS, + **overrides, + ) + ) diff --git a/marker_experiments/caps_grid.py b/marker_experiments/caps_grid.py new file mode 100644 index 00000000..4dee889d --- /dev/null +++ b/marker_experiments/caps_grid.py @@ -0,0 +1,150 @@ +"""Do caps codes pay for themselves? + +Same duplication argument as the leading space, applied to case: without caps codes a +vocabulary holds 'The' and 'the', 'NASA' and 'nasa' as separate entries. With them, a +title-case span is a shift code plus the lowercased form, so the pieces are shared. + +The cost is one extra token per capitalised span, and sentence-initial capitals are very +frequent, so this can easily come out negative. Section 5.3 already showed that reclaiming +vocabulary does not automatically buy compression: removing a 3.17% digit-variant tax was +worth +0.33pp. + +en, 250M characters, 32,768 additional vocabulary, BPE, evaluation withheld from training +-- the same setup as the digit axis, so the plain and bnd_wpd cells there are directly +comparable and are reused. +""" + +import json +import os +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +from script_bpe.corpus.base import PretokenizedCorpus +from script_bpe.pretokenize.scriptencoding import ScriptEncodingV3 +from script_bpe.tokenizers.bpe.trainer import BPETrainer, BPETrainerConfig + +import finewiki1gb_grid as G +from boundary_pretokenizer import BoundaryScriptPretokenizer, BoundaryScriptPretokenizerConfig +from finewiki1gb_grid import CORPORA, NUM_WORKERS, VOCAB, analyse_vocab, commit_cell, ensure_eval, log, train_batches + +LANG = "en" +CHARS = 250_000_000 +RESULT_PATH = os.path.join(HERE, "caps_result.json") +TOKENIZERS = os.path.join(HERE, "tokenizers") + +VARIANTS = { + "bnd_wpd": dict(caps_codes=False), + "bnd_wpd_caps": dict(caps_codes=True), +} + + +def make_pt(**kw): + return BoundaryScriptPretokenizer( + BoundaryScriptPretokenizerConfig( + script_config=ScriptEncodingV3, boundary_targets=("word", "punct", "digit"), **kw + ) + ) + + +def case_stats(tokenizer, pt): + """Vocabulary spent on case variants of the same word. + + Counts entries whose decoded text has a distinct-cased counterpart also in the + vocabulary ('The'/'the'), which is the case analogue of the ' X'/'X' pair count. + """ + marker = getattr(pt, "marker_token_id", None) + codes = {pt.shift_token_id, pt.caps_token_id} - {None} + texts = {} + for t in tokenizer.tokens.values(): + ids = [x for x in t.atomic_tokens if x != marker and x not in codes] + if not ids: + continue + txt = pt.try_decode_strict(ids) + if txt and txt.isalpha(): + texts.setdefault(txt, 0) + texts[txt] += 1 + pairs = 0 + seen = set() + for txt in texts: + if txt.islower() or txt in seen: + continue + low = txt.lower() + if low != txt and low in texts: + pairs += 1 + seen.add(txt) + seen.add(low) + return { + "alpha_entries": len(texts), + "case_dup_pairs": pairs, + "case_dup_vocab_frac": 2 * pairs / len(tokenizer.tokens), + } + + +def main(): + os.makedirs(TOKENIZERS, exist_ok=True) + G.CHARS_PER_LANG = CHARS + results = json.load(open(RESULT_PATH)) if os.path.exists(RESULT_PATH) else {} + eval_texts = ensure_eval(LANG) + eval_chars = sum(map(len, eval_texts)) + log(f"eval: {len(eval_texts)} docs, {eval_chars:,} chars") + + for tag, kw in VARIANTS.items(): + key = f"{LANG}_{tag}" + if key in results: + log(f"{key}: done, skipping") + continue + pt = make_pt(**kw) + corpus_name = f"caps250_{LANG}_{tag}" + try: + corpus = PretokenizedCorpus(name=corpus_name, base_path=CORPORA, pretokenizer=pt) + except FileNotFoundError: + t = time.time() + corpus = PretokenizedCorpus.from_text_batches( + name=corpus_name, base_path=CORPORA, pretokenizer=pt, + text_batches=train_batches(LANG), num_workers=NUM_WORKERS, + ) + log(f"{key}: corpus built in {time.time()-t:.0f}s " + f"unique_chunks={corpus.metadata.get('unique_chunks'):,}") + + t = time.time() + tokenizer = BPETrainer( + pt, corpus, BPETrainerConfig(additional_vocab_size=VOCAB, num_workers=NUM_WORKERS) + ).train() + train_time = time.time() - t + # Prefix with the corpus tag. Without it this 250M-char cell writes + # en_bnd_wpd_bpe_32k.json.gz, the same path the 1 GB grid uses, and silently + # replaces that artifact with a smaller-corpus tokenizer of the same name. + out = os.path.join(TOKENIZERS, f"{corpus_name}_bpe_32k.json.gz") + tokenizer.save(out) + + toks = fails = 0 + for text in eval_texts: + ids = tokenizer.encode(text) + toks += len(ids) + if tokenizer.decode(ids) != text: + fails += 1 + + results[key] = { + "lang": LANG, "variant": tag, "caps_codes": kw["caps_codes"], + "atomic_vocab": len(pt.atomic_tokens), "vocab_size": len(tokenizer.tokens), + "train_seconds": round(train_time), + "unique_chunks": corpus.metadata.get("unique_chunks"), + "eval_chars": eval_chars, "eval_tokens": toks, + "eval_chars_per_token": eval_chars / toks, + "roundtrip_failures": fails, + **analyse_vocab(tokenizer, pt), **case_stats(tokenizer, pt), + } + with open(RESULT_PATH, "w") as f: + json.dump(results, f, indent=2) + log(f" {key}: {eval_chars/toks:.4f} ch/tok case_pairs=" + f"{results[key]['case_dup_pairs']} {round(train_time)}s rt={fails}") + commit_cell(key) + + log(f"DONE: {len(results)} cells") + + +if __name__ == "__main__": + main() diff --git a/marker_experiments/caps_result.json b/marker_experiments/caps_result.json new file mode 100644 index 00000000..f09fa18f --- /dev/null +++ b/marker_experiments/caps_result.json @@ -0,0 +1,44 @@ +{ + "en_bnd_wpd": { + "lang": "en", + "variant": "bnd_wpd", + "caps_codes": false, + "atomic_vocab": 1711, + "vocab_size": 34479, + "train_seconds": 92, + "unique_chunks": 906491, + "eval_chars": 2219281, + "eval_tokens": 571265, + "eval_chars_per_token": 3.884853789397215, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 15021, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 212, + "whitespace_only_vocab_entries": 28, + "alpha_entries": 26661, + "case_dup_pairs": 5017, + "case_dup_vocab_frac": 0.29101772093158157 + }, + "en_bnd_wpd_caps": { + "lang": "en", + "variant": "bnd_wpd_caps", + "caps_codes": true, + "atomic_vocab": 1713, + "vocab_size": 34481, + "train_seconds": 92, + "unique_chunks": 906765, + "eval_chars": 2219281, + "eval_tokens": 571502, + "eval_chars_per_token": 3.8832427533062, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 15051, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.00029001479075432845, + "marker_variant_extra_slots": 214, + "whitespace_only_vocab_entries": 28, + "alpha_entries": 21819, + "case_dup_pairs": 91, + "case_dup_vocab_frac": 0.005278269191728778 + } +} \ No newline at end of file diff --git a/marker_experiments/digit_split_grid.py b/marker_experiments/digit_split_grid.py new file mode 100644 index 00000000..88e5d6ec --- /dev/null +++ b/marker_experiments/digit_split_grid.py @@ -0,0 +1,208 @@ +"""Does splitting digits remove the digit-variant tax? + +Measured on the 1 GB grid, delimiting whole digit runs costs English 1,093 vocabulary +slots (3.17%): every distinct number acquires up to four marked forms, so bnd_wpd spends +more entries on numbers than the baseline while covering fewer than half as many. + +digit_handling bounds the markable set, because only a run's first and last GROUP can +carry a marker: + + None every distinct number is markable (what the 1 GB grid used) + SPLIT 10 markable strings + RTL3 1110 markable strings (pretokenizer.py registers exactly 1000+100+10) + +This runs en at 1 GB, 32,768 additional vocabulary, BPE, over +{plain, bnd_wpd} x {None, SPLIT, RTL3}. The None cells already exist in the main grid +and are reused. + +Fairness note: ScriptPretokenizer.decode has no path for digit-group tokens, so the +stock baseline cannot round-trip with digit_handling set at all (digit_handling was +only ever exercised with UTF8Pretokenizer). The baseline used here is the stock one plus +exactly that decode fix and nothing else, so both sides of the comparison are equally +able to use digit splitting. +""" + +import json +import os +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +from script_bpe.corpus.base import PretokenizedCorpus +from script_bpe.pretokenize.pretokenizer import ScriptPretokenizer, ScriptPretokenizerConfig +from script_bpe.pretokenize.scriptencoding import ScriptEncodingV3 +from script_bpe.tokenizers.bpe.trainer import BPETrainer, BPETrainerConfig + +from boundary_pretokenizer import BoundaryScriptPretokenizer, BoundaryScriptPretokenizerConfig +import finewiki1gb_grid as G +from finewiki1gb_grid import CORPORA, VOCAB, NUM_WORKERS, analyse_vocab, commit_cell, ensure_eval, log, train_batches + +LANG = "en" +# 250M chars, not the 1 GB of the main grid: the digit-variant tax is a property of the +# markable set, not of scale, and at 1 GB each cell needed a 716s corpus build that the +# container's ~30-60 min working-tree wipes kept destroying mid-flight. All six cells here +# are rebuilt at this size so the three digit_handling settings are directly comparable to +# each other; they are NOT comparable to the 1 GB numbers in the main grid. +DIGIT_AXIS_CHARS = 250_000_000 +RESULT_PATH = os.path.join(HERE, "digit_split_result.json") +TOKENIZERS = os.path.join(HERE, "tokenizers") + + +class DigitAwareScriptPretokenizerConfig(ScriptPretokenizerConfig): + cls: str = "DigitAwareScriptPretokenizer" + + +class DigitAwareScriptPretokenizer(ScriptPretokenizer, config_type=DigitAwareScriptPretokenizerConfig): + """Stock baseline plus the two fixes `digit_handling` needs on ScriptPretokenizer. + + ScriptPretokenizer does not support digit_handling at all -- it was only ever + exercised with UTF8Pretokenizer -- and fails in two independent places: + + * split_encoded raises ValueError on any chunk that is not entirely ScriptCharEnc, + and a digit chunk is entirely DigitsEnc; + * decode has no path for digit-group tokens, so every digit becomes U+FFFD. + + Both are fixed here and nothing else differs from scriptenc3_cb, so the baseline and + the boundary variant are equally able to use digit splitting. + """ + + def __init__(self, config): + super().__init__(config) + self.digit_token_ids = {tid for tid, txt in self.atomic_tokens.items() if txt.isdigit()} + + def split_encoded(self, encoding): + if any(getattr(c, "script_id", 0) == -1 for c in encoding): + return [encoding] # a digit chunk is already its own pretoken + return super().split_encoded(encoding) + + def decode(self, tokenization, errors="replace") -> str: + decoded = "" + i = 0 + n = len(tokenization) + while i < n: + if tokenization[i] in self.digit_token_ids: + decoded += self.atomic_tokens[tokenization[i]] + i += 1 + continue + script_tok = tokenization[i] + ix_tok = tokenization[i + 1] if i + 1 < n else None + if (script_tok, ix_tok) in self.detokenize_map: + decoded += self.detokenize_map[(script_tok, ix_tok)] + i += 2 + else: + if errors == "backslashreplace": + decoded += self.atomic_tokens[script_tok] + elif errors == "replace": + decoded += "�" + elif errors == "strict": + raise ValueError(f"Invalid tokenization: ({script_tok}, {ix_tok})") + else: + raise ValueError(f"Unknown error handling mode: {errors}") + i += 1 + return decoded + + +def make_pt(tag, digit_handling): + if tag == "plain": + return DigitAwareScriptPretokenizer( + DigitAwareScriptPretokenizerConfig( + script_config=ScriptEncodingV3, enforce_char_boundaries=True, digit_handling=digit_handling + ) + ) + return BoundaryScriptPretokenizer( + BoundaryScriptPretokenizerConfig( + script_config=ScriptEncodingV3, + boundary_targets=("word", "punct", "digit"), + digit_handling=digit_handling, + ) + ) + + +def digit_stats(tokenizer, pt): + """Pure-digit vocabulary entries, distinct numbers covered, slots lost to variants.""" + m = getattr(pt, "marker_token_id", None) + forms = {} + entries = 0 + for t in tokenizer.tokens.values(): + ids = list(t.atomic_tokens) + core = [x for x in ids if x != m] if m is not None else ids + if not core: + continue + txt = pt.try_decode_strict(core) + if not txt or not txt.isdigit(): + continue + entries += 1 + key = ("<|>" if m is not None and ids[0] == m else "") + txt + ("<|>" if m is not None and ids[-1] == m else "") + forms.setdefault(txt, set()).add(key) + return { + "pure_digit_entries": entries, + "distinct_numbers": len(forms), + "digit_variant_extra_slots": sum(len(v) - 1 for v in forms.values()), + } + + +def main(): + os.makedirs(TOKENIZERS, exist_ok=True) + results = json.load(open(RESULT_PATH)) if os.path.exists(RESULT_PATH) else {} + G.CHARS_PER_LANG = DIGIT_AXIS_CHARS + eval_texts = ensure_eval(LANG) + eval_chars = sum(map(len, eval_texts)) + + G.CHARS_PER_LANG = DIGIT_AXIS_CHARS # applies to train_batches/ensure_eval below + for digit_handling in ["None", "SPLIT", "RTL3"]: + for tag in ["plain", "bnd_wpd"]: + key = f"{LANG}_{tag}_{digit_handling}" + if key in results: + log(f"{key}: done, skipping") + continue + pt = make_pt(tag, None if digit_handling == "None" else digit_handling) + corpus_name = f"digitsplit250_{LANG}_{tag}_{digit_handling}" + try: + corpus = PretokenizedCorpus(name=corpus_name, base_path=CORPORA, pretokenizer=pt) + except FileNotFoundError: + t = time.time() + corpus = PretokenizedCorpus.from_text_batches( + name=corpus_name, base_path=CORPORA, pretokenizer=pt, + text_batches=train_batches(LANG), num_workers=NUM_WORKERS, + ) + log(f"{key}: corpus built in {time.time()-t:.0f}s " + f"unique_chunks={corpus.metadata.get('unique_chunks'):,}") + + t = time.time() + tokenizer = BPETrainer( + pt, corpus, BPETrainerConfig(additional_vocab_size=VOCAB, num_workers=NUM_WORKERS) + ).train() + train_time = time.time() - t + out = os.path.join(TOKENIZERS, f"{key}_bpe_32k.json.gz") + tokenizer.save(out) + + toks = fails = 0 + for text in eval_texts: + ids = tokenizer.encode(text) + toks += len(ids) + if tokenizer.decode(ids) != text: + fails += 1 + + results[key] = { + "lang": LANG, "pretokenizer": tag, "digit_handling": digit_handling, + "atomic_vocab": len(pt.atomic_tokens), "vocab_size": len(tokenizer.tokens), + "train_seconds": round(train_time), + "unique_chunks": corpus.metadata.get("unique_chunks"), + "eval_chars": eval_chars, "eval_tokens": toks, + "eval_chars_per_token": eval_chars / toks, + "roundtrip_failures": fails, + **analyse_vocab(tokenizer, pt), **digit_stats(tokenizer, pt), + } + with open(RESULT_PATH, "w") as f: + json.dump(results, f, indent=2) + log(f" {key}: {eval_chars/toks:.4f} ch/tok digit_variants=" + f"{results[key]['digit_variant_extra_slots']} {round(train_time)}s rt={fails}") + commit_cell(key) + + log(f"DONE: {len(results)} cells") + + +if __name__ == "__main__": + main() diff --git a/marker_experiments/digit_split_result.json b/marker_experiments/digit_split_result.json new file mode 100644 index 00000000..72be1d08 --- /dev/null +++ b/marker_experiments/digit_split_result.json @@ -0,0 +1,128 @@ +{ + "en_plain_None": { + "lang": "en", + "pretokenizer": "plain", + "digit_handling": "None", + "atomic_vocab": 1710, + "vocab_size": 34478, + "train_seconds": 95, + "unique_chunks": 1020944, + "eval_chars": 2219281, + "eval_tokens": 590550, + "eval_chars_per_token": 3.757990009313352, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27667, + "space_dup_pairs": 3212, + "space_dup_vocab_frac": 0.18632171239631068, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 29, + "pure_digit_entries": 1486, + "distinct_numbers": 1486, + "digit_variant_extra_slots": 0 + }, + "en_bnd_wpd_None": { + "lang": "en", + "pretokenizer": "bnd_wpd", + "digit_handling": "None", + "atomic_vocab": 1711, + "vocab_size": 34479, + "train_seconds": 92, + "unique_chunks": 906491, + "eval_chars": 2219281, + "eval_tokens": 571265, + "eval_chars_per_token": 3.884853789397215, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 15021, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 212, + "whitespace_only_vocab_entries": 28, + "pure_digit_entries": 1691, + "distinct_numbers": 608, + "digit_variant_extra_slots": 1083 + }, + "en_plain_SPLIT": { + "lang": "en", + "pretokenizer": "plain", + "digit_handling": "SPLIT", + "atomic_vocab": 1720, + "vocab_size": 34488, + "train_seconds": 92, + "unique_chunks": 972588, + "eval_chars": 2219281, + "eval_tokens": 644453, + "eval_chars_per_token": 3.4436661789145213, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 28969, + "space_dup_pairs": 3375, + "space_dup_vocab_frac": 0.19572025052192066, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 29, + "pure_digit_entries": 14, + "distinct_numbers": 14, + "digit_variant_extra_slots": 0 + }, + "en_bnd_wpd_SPLIT": { + "lang": "en", + "pretokenizer": "bnd_wpd", + "digit_handling": "SPLIT", + "atomic_vocab": 1721, + "vocab_size": 34489, + "train_seconds": 115, + "unique_chunks": 844986, + "eval_chars": 2219281, + "eval_tokens": 621408, + "eval_chars_per_token": 3.5713750064369947, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 15951, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002899475194989707, + "marker_variant_extra_slots": 217, + "whitespace_only_vocab_entries": 29, + "pure_digit_entries": 63, + "distinct_numbers": 14, + "digit_variant_extra_slots": 34 + }, + "en_plain_RTL3": { + "lang": "en", + "pretokenizer": "plain", + "digit_handling": "RTL3", + "atomic_vocab": 2820, + "vocab_size": 35588, + "train_seconds": 122, + "unique_chunks": 973688, + "eval_chars": 2219281, + "eval_tokens": 598249, + "eval_chars_per_token": 3.7096275965358907, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 28969, + "space_dup_pairs": 3375, + "space_dup_vocab_frac": 0.1896706755085984, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 29, + "pure_digit_entries": 1114, + "distinct_numbers": 1114, + "digit_variant_extra_slots": 0 + }, + "en_bnd_wpd_RTL3": { + "lang": "en", + "pretokenizer": "bnd_wpd", + "digit_handling": "RTL3", + "atomic_vocab": 2821, + "vocab_size": 35589, + "train_seconds": 120, + "unique_chunks": 849383, + "eval_chars": 2219281, + "eval_tokens": 578766, + "eval_chars_per_token": 3.834504791228234, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 15119, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.00028098569782798055, + "marker_variant_extra_slots": 214, + "whitespace_only_vocab_entries": 28, + "pure_digit_entries": 2607, + "distinct_numbers": 1114, + "digit_variant_extra_slots": 1478 + } +} \ No newline at end of file diff --git a/marker_experiments/downstream/.gitignore b/marker_experiments/downstream/.gitignore new file mode 100644 index 00000000..fe8c3523 --- /dev/null +++ b/marker_experiments/downstream/.gitignore @@ -0,0 +1,5 @@ +# Cluster-generated: matched tokenizers (~15 MB each) and their corpus cache. +# manifest.json is deliberately NOT ignored -- it is the record of what was trained +# (vocabulary sizes, train times, chars/token) and is what shows the arms were matched. +tokenizers/ +corpora/ diff --git a/marker_experiments/downstream/README.md b/marker_experiments/downstream/README.md new file mode 100644 index 00000000..ad2ae510 --- /dev/null +++ b/marker_experiments/downstream/README.md @@ -0,0 +1,283 @@ +# Downstream LM evaluation for boundary-marker tokenizers + +Boundary markers buy **+2.14 % chars/token** on average over the SCRIPT-v3 baseline +across six languages (English +3.77 %), and caps codes reclaim **29 % of the +vocabulary** at no compression cost. Neither result says anything about language +modelling. This directory runs that check: for each tokenizer, pretrain a nanochat base +model and report **DCLM CORE** and **validation bits-per-byte**. + +Everything here rides on the existing harness — `eval/py-nanochat` (`pynanochat`) and +`paper_utils/hybrid/downstream/run_downstream_eval.py`, the same path the MinGram +downstream table used. What this directory adds is the four things that path needs +before it can accept a boundary tokenizer: + +| file | what it does | +|---|---| +| `boundary_tokenizer.py` | makes boundary tokenizers loadable in a fresh process | +| `train_matched.py` | trains one **vocabulary-matched** tokenizer per arm | +| `smoke_test.py` | every check that does not need a GPU | +| `run_arms.sh` | train → check → run → collect, for one arm × seed set | +| `collect_results.py` | parse run logs into a TSV | + +## What has been verified, and what has not + +Verified locally, on CPU, on all 56 existing tokenizers plus a freshly trained matched +set (`smoke_test.py`, 0 failures): + +- fresh-process load by dotted class path, for both BPE and MinGram serialisations +- the `pynanochat.Tokenizer` contract, dense-id space, synthetic BOS at `n`, `vocab = n+1` +- round-trip through the adapter on marker-, caps-, digit- and mixed-script text, + and on `tests/data/taylorswift.txt` +- batch encode agrees with single encode (`pretokenize.py` uses the batch path) +- the `vocab <= 65535` uint16 bound +- matched vocabulary across arms, as a hard failure + +**Not verified:** the GPU leg — nanochat pretrain, CORE, bpb. The development container +has neither `torch` nor a GPU, so `write_token_bytes` (which uses `torch.save`) and +everything downstream of it are untested. `SMOKE=1` below is the first thing to run on +the cluster, and it exercises exactly that leg. + +## Why a separate training step + +The compression grid fixed `additional_vocab_size = 32768` for every arm, so the arms +end up with *different total* vocabularies — the boundary marker and the two caps codes +are extra atomic tokens: + +``` +plain 1710 atomic + 32768 = 34478 +bnd_wpd 1711 atomic + 32768 = 34479 +bnd_wpd_caps 1713 atomic + 32768 = 34481 +``` + +That is the right control for measuring compression: every arm gets the same number of +*learned* merges. It is the wrong control downstream, where vocabulary size sets the +embedding and unembedding shapes, hence the parameter count, hence nanochat's +compute-optimal token horizon. `train_matched.py` matches the total instead and lets the +learned budget absorb the difference: + +``` +additional_vocab_size = total_vocab - len(pretokenizer.atomic_tokens) +``` + +The default total is **34,685**, the matched vocabulary of the MinGram downstream table, +so these runs sit on the same axis as that table. A three-token difference in 34 k would +not move CORE, but it is free to get exactly right, and `train_matched.py` exits non-zero +if the arms do not agree. + +Do **not** reuse `marker_experiments/tokenizers/*.json.gz` for this. They are the +compression grid's tokenizers: unmatched by construction, and trained on FineWiki rather +than web text. + +## Why `boundary_tokenizer.py` exists + +`Pretokenizer.REGISTRY` is filled by `__init_subclass__`, so it only contains +`BoundaryScriptPretokenizer` in a process that imported the defining module. The harness +loads tokenizers in fresh subprocesses, where: + +```python +BPETokenizer.load("..._bnd_wpd_bpe.json.gz") +# KeyError: 'BoundaryScriptPretokenizer' +``` + +`marker_experiments/downstream/boundary_tokenizer.py` imports the module for its +registration side effect and re-exports the tokenizer classes **unchanged** — +`BoundaryBPETokenizer is BPETokenizer`, no subclass, no behaviour change. So the fix is +one flag: + +``` +--tokenizer-class marker_experiments.downstream.boundary_tokenizer.BoundaryBPETokenizer +--tokenizer-class marker_experiments.downstream.boundary_tokenizer.BoundaryMinGramModel +``` + +Use the MinGram spelling for MinGram-trained models; `BPETokenizer.load` on one raises +`KeyError: 'merge_rules'`. + +For the child process to import that path, the repo must be installed **editable** +(`uv sync` does this) — the child runs with `cwd` set to the nanochat clone, and picks up +the repo root from the editable install's `.pth`, not from `cwd`. + +## Setup + +This work lives on a branch, not on `main`. Clone that branch: + +```bash +git clone -b claude/fineweb-space-neighbors-k10ufw \ + https://github.com/sanderland/script_tok.git +cd script_tok +``` + +(Branch `claude/fineweb-space-neighbors-k10ufw`, draft PR +[#7](https://github.com/sanderland/script_tok/pull/7). If you already have the repo: +`git fetch origin claude/fineweb-space-neighbors-k10ufw && git checkout claude/fineweb-space-neighbors-k10ufw`.) + +Then the environment. The **editable** install is not optional: the eval's child +processes run with `cwd` set to the nanochat clone and find `marker_experiments.*` +through the editable install's `.pth`, not through `cwd`. + +```bash +uv sync --extra downstream # editable script_bpe + pynanochat, torch, deps +uv pip install "nanochat @ git+https://github.com/karpathy/nanochat" + +# pynanochat shells into a vendored clone; the runner errors without it +git clone https://github.com/karpathy/nanochat eval/py-nanochat/vendor/nanochat +``` + +Verify the setup before asking for a GPU. This needs no GPU and takes seconds: + +```bash +uv run python marker_experiments/downstream/smoke_test.py +``` + +Expect `0 failure(s)` over the checked-in compression tokenizers. It will note that +their vocabularies are unmatched — correct, and `train_matched.py` is what fixes it. +`write_token_bytes` reports `[ok]` here and `[skip]` without torch; on the cluster it +must be `[ok]`. + +Hardware and budget: one H100 80 GB per job at `--depth 12`. `--depth 24` (what the +MinGram table used) needs more; drop `--device-batch-size` to 16/8/4 on OOM. Tokenizer +training wants ~90 CPUs and is a one-off shared across seeds. Disk: ~50 GB under +`$NANOCHAT_BASE` for shards and checkpoints, plus ~15 MB per tokenizer. + +## Run + +Start with the pipeline check. It is minutes, CORE comes out near random, and the point +is only that download → inject → train → eval → parse works for these tokenizers: + +```bash +SMOKE=1 ARMS=plain,bnd_wpd DEPTH=12 marker_experiments/downstream/run_arms.sh +``` + +Then the real thing: + +```bash +ARMS=plain,bnd_wpd,bnd_wpd_caps \ +SEEDS=0,1,2 \ +DEPTH=12 \ +TRAIN_WORKERS=90 \ +OUT=results/marker_downstream \ +marker_experiments/downstream/run_arms.sh +``` + +`run_arms.sh` runs three steps: train the matched tokenizers, run `smoke_test.py` against +them (it stops before spending GPU hours if anything is off), then one +`run_downstream_eval.py` per arm × seed, logging to `$OUT/logs/__d_s.log`. +It skips any run whose log already contains a result, so it is resumable after a +pre-emption. + +### Knobs + +| var | default | notes | +|---|---|---| +| `ARMS` | `plain,bnd_wpd,bnd_wpd_caps` | `plain` is the SCRIPT-v3 baseline; also `bnd_w`, `bnd_wp`, and any `*_caps` | +| `SEEDS` | `0` | the MinGram table used 20 seeds per method | +| `TRAINER` | `bpe` | or `mingram` (`f = 1.15`) | +| `CORPUS` | `fineweb_en_5gb` | tokenizer-training corpus; `finewiki_en_1gb` reproduces the compression grid's domain | +| `VOCAB` | `34685` | matched total vocabulary | +| `DEPTH` | `12` | `model_dim = depth * 64` | +| `GPUS` | `1` | `>1` launches torchrun/DDP | +| `NUM_SHARDS` | `8` | train data shards to download | +| `TRAIN_WORKERS` | `nproc` | tokenizer-trainer workers | +| `NANOCHAT_BASE` | `~/.cache/nanochat_marker` | data, token bytes, checkpoints | +| `SMOKE` | `0` | tiny end-to-end run | + +### Slurm + +One arm × seed per job; `run_arms.sh` is idempotent, so an array over seeds works with +the arm list held fixed: + +```bash +#!/bin/bash +#SBATCH --array=0-2 +#SBATCH --gpus=1 --cpus-per-task=90 --time=12:00:00 +export ARMS=plain,bnd_wpd,bnd_wpd_caps +export SEEDS=$SLURM_ARRAY_TASK_ID +export TRAIN_WORKERS=90 +export OUT=results/marker_downstream +srun marker_experiments/downstream/run_arms.sh +``` + +Tokenizer training is shared across seeds and cached by output path, so only the first +job in the array pays for it. If jobs start simultaneously they will each build the +pretokenized corpus; run step 1 alone once first to avoid that: + +```bash +uv run python marker_experiments/downstream/train_matched.py \ + --arms plain,bnd_wpd,bnd_wpd_caps --workers 90 \ + --eval-texts marker_experiments/eval_texts/en.json +``` + +That prints chars/token on the compression grid's held-out English slice, which is the +cheapest way to confirm the tokenizers came out as expected before any GPU time. + +### Many seeds: pre-tokenize once + +`script_bpe.encode` is pure Python, and with 20 seeds per arm the corpus gets encoded 20 +times identically. `paper_utils/hybrid/downstream/pretokenize.py` encodes it once to +uint16 shards that `pynanochat.pretok_dataloader` reads: + +```bash +uv run python paper_utils/hybrid/downstream/pretokenize.py \ + --tokenizer-path marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_bpe_v34685.json.gz \ + --tokenizer-class marker_experiments.downstream.boundary_tokenizer.BoundaryBPETokenizer \ + --base-dir $NANOCHAT_BASE --out-dir nc_runs/pretok/bnd_wpd --workers 90 +``` + +`run_arms.sh` does not wire this in — it uses the on-the-fly path with +`encode_workers`, which is self-contained and is what the smoke run exercises. + +## Results + +```bash +uv run python marker_experiments/downstream/collect_results.py \ + --logs-dir results/marker_downstream/logs --out results/marker_downstream/results.tsv +``` + +`pynanochat.run_experiment` returns an `ExperimentResult` but persists nothing, so the +logs are the record. The TSV columns (`method`, `seed`, `val_bpb`, `train_bpb`, `core`, +plus `task_*`) match what `paper_utils/hybrid/downstream_results.py` expects, so the +existing table generators can read it. Logs without a result block — pre-empted or OOM +jobs — are listed and skipped, and re-running `run_arms.sh` picks them up. + +`collect_results.py` also prints a per-arm summary, so a finished sweep is readable +without opening the TSV: + +``` + arm n val_bpb CORE + bnd_wpd 3 0.9876 0.1234 + plain 3 0.9990 0.1100 +``` + +### What to send back + +`$OUT/results.tsv`, `$OUT/logs/`, and `marker_experiments/downstream/manifest.json` (the +per-arm vocabulary sizes, train times, and chars/token — it is what shows the arms were +genuinely matched). The tokenizers themselves are reproducible from the manifest and need +not travel. Committing straight to the branch is fine; `tokenizers/` and `corpora/` are +gitignored, the TSV and manifest are not large. + +## What to expect + +The compression side of this work found, three times over, that reclaiming vocabulary +does not convert into compression: unifying `' the'`/`'the'` and `'The'`/`'the'` frees +thousands of slots, and BPE spends them on things worth almost nothing. The open +question is whether it converts into *modelling* quality, which compression cannot +answer. Two outcomes are informative: + +- **bpb improves roughly in line with the +3.77 % English compression gain** — the gain + is real and the vocabulary reclamation was worth doing. +- **bpb is flat while compression improved** — the marker moved token boundaries without + making the sequence more predictable, and the compression gain is bookkeeping. + +CORE at depth 12 is noisy; the MinGram table needed 20 seeds per method for +significance. Treat a 3-seed run as a direction, not a result. + +## Note on the checked-in compression tokenizers + +`marker_experiments/tokenizers/` holds the compression grid's tokenizers. Two files there +were originally written by the 1 GB grid and then silently overwritten by the 250 M-char +caps grid, which used the same filenames; they have been renamed to `caps250_*` to match +what they actually are, and `caps_grid.py` now prefixes its outputs. The 1 GB +`en_bnd_wpd` artifact is gone as a result — the 1 GB *results* in +`finewiki1gb_result.json` are unaffected (that cell recorded `unique_chunks=2,072,665`, +against 906,491 for the 250 M corpus), and nothing here depends on the artifact, since +every downstream arm is retrained by `train_matched.py`. diff --git a/marker_experiments/downstream/boundary_tokenizer.py b/marker_experiments/downstream/boundary_tokenizer.py new file mode 100755 index 00000000..e0bc24dc --- /dev/null +++ b/marker_experiments/downstream/boundary_tokenizer.py @@ -0,0 +1,34 @@ +"""Loadable entry points for boundary-marker tokenizers. + +`Pretokenizer.REGISTRY` is populated by `__init_subclass__`, so it only knows about +`BoundaryScriptPretokenizer` in a process that has imported the module defining it. +The downstream harness loads a tokenizer by dotted class path in a fresh subprocess: + + python -c "from script_bpe.tokenizers.bpe import BPETokenizer; \ + BPETokenizer.load('..._bnd_wpd_bpe_32k.json.gz')" + KeyError: 'BoundaryScriptPretokenizer' + +Importing *this* module registers the pretokenizer as a side effect, and it re-exports +the tokenizer classes unchanged, so pointing the harness here fixes the load with no +subclassing and no behaviour change: + + --tokenizer-class marker_experiments.downstream.boundary_tokenizer.BoundaryBPETokenizer + +The aliases are the real classes, not subclasses: `BoundaryBPETokenizer is BPETokenizer`. +Nothing about encoding or decoding lives here -- only the import that makes the +registry complete. The same trick covers the baseline arm, which needs no registration +but is exported so every arm can use one uniform `--tokenizer-class` spelling. +""" + +from script_bpe.tokenizers.bpe import BPETokenizer +from script_bpe.tokenizers.mingram.model import MinGramModel + +# The import is the point: it runs __init_subclass__ and fills REGISTRY. +from marker_experiments.boundary_pretokenizer import BoundaryScriptPretokenizer + +assert "BoundaryScriptPretokenizer" in BoundaryScriptPretokenizer.REGISTRY + +BoundaryBPETokenizer = BPETokenizer +BoundaryMinGramModel = MinGramModel + +__all__ = ["BoundaryBPETokenizer", "BoundaryMinGramModel", "BPETokenizer", "MinGramModel"] diff --git a/marker_experiments/downstream/collect_results.py b/marker_experiments/downstream/collect_results.py new file mode 100755 index 00000000..1833d45c --- /dev/null +++ b/marker_experiments/downstream/collect_results.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Parse run_downstream_eval.py logs into one TSV. + +`pynanochat.run_experiment` returns an `ExperimentResult` but persists nothing, and +`run_downstream_eval.py` only prints it, so the logs are the record. This reads the +result block each run prints: + + ============================================================ + Downstream eval result + ============================================================ + tokenizer_id : bnd_wpd_bpe_d12_s0 + depth : 12 + vocab_size : 34,686 + CORE metric : 0.1234 + val bpb : 0.9876 + train bpb : 0.9800 + artifact_dir : ... + per-task CORE (centered): + hellaswag_zeroshot 0.0421 + ... + +Columns match paper_utils/hybrid/downstream_results.py's expectations (method, seed, +val_bpb, train_bpb, core) so the existing table generators can read this file, with +per-task CORE in the remaining columns. + + uv run python marker_experiments/downstream/collect_results.py \ + --logs-dir results/marker_downstream/logs --out results/marker_downstream/results.tsv +""" + +import csv +import os +import re + +import cyclopts + +app = cyclopts.App() + +SCALARS = { + "tokenizer_id": "tokenizer_id", + "depth": "depth", + "vocab_size": "vocab_size", + "CORE metric": "core", + "val bpb": "val_bpb", + "train bpb": "train_bpb", +} +# tokenizer_id is __d_s, e.g. bnd_wpd_caps_bpe_d12_s0 +TAG_RE = re.compile(r"^(?P.+)_(?Pbpe|mingram)_d(?P\d+)_s(?P\d+)$") + + +def parse_log(path): + """Return one row, or None if the run did not reach a result block.""" + with open(path, errors="replace") as f: + lines = f.read().splitlines() + try: + start = max(i for i, l in enumerate(lines) if l.strip() == "Downstream eval result") + except ValueError: + return None + + row, in_tasks = {}, False + for line in lines[start:]: + stripped = line.strip() + if stripped.startswith("per-task CORE"): + in_tasks = True + continue + if in_tasks: + parts = stripped.split() + if len(parts) == 2: + row[f"task_{parts[0]}"] = parts[1] + continue + key, _, value = stripped.partition(":") + col = SCALARS.get(key.strip()) + if col: + row[col] = value.strip().replace(",", "") + + if "core" not in row and "val_bpb" not in row: + return None + tag = row.get("tokenizer_id", os.path.basename(path).removesuffix(".log")) + m = TAG_RE.match(tag) + if m: + # `method` is what the paper's table generators key on. + row["method"] = f"{m['arm']}_{m['trainer']}" + row["arm"] = m["arm"] + row["trainer"] = m["trainer"] + row["seed"] = m["seed"] + else: + row["method"] = tag + row["log"] = os.path.basename(path) + return row + + +@app.default +def main(logs_dir: str, out: str = "results.tsv") -> None: + """Collect every finished run under `logs_dir` into `out`. + + Args: + logs_dir: Directory of .log files written by run_arms.sh. + out: TSV to write. + """ + rows, skipped = [], [] + for name in sorted(os.listdir(logs_dir)): + if not name.endswith(".log"): + continue + row = parse_log(os.path.join(logs_dir, name)) + (rows.append(row) if row else skipped.append(name)) + + if not rows: + raise SystemExit(f"no finished runs in {logs_dir} ({len(skipped)} incomplete)") + + lead = ["method", "arm", "trainer", "seed", "val_bpb", "train_bpb", "core", + "depth", "vocab_size", "tokenizer_id", "log"] + tasks = sorted({k for r in rows for k in r if k.startswith("task_")}) + fields = [c for c in lead if any(c in r for r in rows)] + tasks + + os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True) + with open(out, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields, delimiter="\t", extrasaction="ignore") + w.writeheader() + w.writerows(rows) + print(f"[collect] {len(rows)} runs -> {out}") + if skipped: + print(f"[collect] {len(skipped)} log(s) without a result block: {', '.join(skipped)}") + + # Per-arm summary, so a cluster run is readable without loading the TSV. + by_arm = {} + for r in rows: + if r.get("val_bpb"): + by_arm.setdefault(r.get("arm", r["method"]), []).append( + (float(r["val_bpb"]), float(r["core"]) if r.get("core") else None) + ) + if by_arm: + print(f"\n {'arm':<16} {'n':>2} {'val_bpb':>9} {'CORE':>8}") + for arm, vals in sorted(by_arm.items(), key=lambda kv: sum(v[0] for v in kv[1]) / len(kv[1])): + bpb = sum(v[0] for v in vals) / len(vals) + cores = [v[1] for v in vals if v[1] is not None] + core = f"{sum(cores) / len(cores):8.4f}" if cores else " -" + print(f" {arm:<16} {len(vals):>2} {bpb:9.4f} {core}") + + +if __name__ == "__main__": + app() diff --git a/marker_experiments/downstream/run_arms.sh b/marker_experiments/downstream/run_arms.sh new file mode 100755 index 00000000..059ab296 --- /dev/null +++ b/marker_experiments/downstream/run_arms.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Downstream LM comparison for boundary-marker tokenizers, one arm x seed per GPU job. +# +# Everything is driven by environment variables so the same script works interactively, +# under `srun`, or as a Slurm array (see README). Each run appends a log under $OUT/logs; +# collect_results.py turns those logs into a TSV. +# +# ARMS=plain,bnd_wpd,bnd_wpd_caps SEEDS=0,1,2 DEPTH=12 ./run_arms.sh +# +# Set SMOKE=1 for the ~minutes-long pipeline check (CORE will be near random; the point +# is that download -> inject -> train -> eval -> parse all work for these tokenizers). + +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO" + +ARMS=${ARMS:-plain,bnd_wpd,bnd_wpd_caps} +SEEDS=${SEEDS:-0} +TRAINER=${TRAINER:-bpe} +CORPUS=${CORPUS:-fineweb_en_5gb} +VOCAB=${VOCAB:-34685} +DEPTH=${DEPTH:-12} +GPUS=${GPUS:-1} +NUM_SHARDS=${NUM_SHARDS:-8} +TRAIN_WORKERS=${TRAIN_WORKERS:-$(nproc)} +SMOKE=${SMOKE:-0} +OUT=${OUT:-results/marker_downstream} +NANOCHAT_BASE=${NANOCHAT_BASE:-$HOME/.cache/nanochat_marker} + +TOK_DIR="marker_experiments/downstream/tokenizers" +DOTTED_BPE=marker_experiments.downstream.boundary_tokenizer.BoundaryBPETokenizer +DOTTED_MINGRAM=marker_experiments.downstream.boundary_tokenizer.BoundaryMinGramModel +if [[ "$TRAINER" == "mingram" ]]; then DOTTED=$DOTTED_MINGRAM; else DOTTED=$DOTTED_BPE; fi + +mkdir -p "$OUT/logs" + +echo "== step 1/3: vocabulary-matched tokenizers (${TRAINER}, vocab ${VOCAB}, ${CORPUS})" +uv run python marker_experiments/downstream/train_matched.py \ + --arms "$ARMS" --trainer "$TRAINER" --corpus "$CORPUS" \ + --total-vocab "$VOCAB" --workers "$TRAIN_WORKERS" \ + --eval-texts marker_experiments/eval_texts/en.json + +echo "== step 2/3: tokenizer-side checks (must be clean before burning GPU hours)" +uv run python marker_experiments/downstream/smoke_test.py \ + --tokenizer-dir "$TOK_DIR" --pattern "_${TRAINER}_v${VOCAB}" + +echo "== step 3/3: downstream runs" +IFS=',' read -ra ARM_LIST <<< "$ARMS" +IFS=',' read -ra SEED_LIST <<< "$SEEDS" +for arm in "${ARM_LIST[@]}"; do + for seed in "${SEED_LIST[@]}"; do + tok="${TOK_DIR}/${CORPUS}_${arm}_${TRAINER}_v${VOCAB}.json.gz" + [[ -f "$tok" ]] || { echo "missing $tok"; exit 1; } + tag="${arm}_${TRAINER}_d${DEPTH}_s${seed}" + log="$OUT/logs/${tag}.log" + if [[ -s "$log" ]] && grep -q "CORE metric" "$log"; then + echo "-- $tag: already done, skipping" + continue + fi + echo "-- $tag -> $log" + smoke_flag=() + [[ "$SMOKE" == "1" ]] && smoke_flag=(--smoke) + uv run python paper_utils/hybrid/downstream/run_downstream_eval.py \ + --tokenizer-path "$tok" \ + --tokenizer-class "$DOTTED" \ + --tokenizer-id "$tag" \ + --depth "$DEPTH" --gpus "$GPUS" --seed "$seed" \ + --num-shards "$NUM_SHARDS" \ + --base-dir "$NANOCHAT_BASE" \ + "${smoke_flag[@]}" 2>&1 | tee "$log" + done +done + +echo "== collecting" +uv run python marker_experiments/downstream/collect_results.py --logs-dir "$OUT/logs" --out "$OUT/results.tsv" diff --git a/marker_experiments/downstream/smoke_test.py b/marker_experiments/downstream/smoke_test.py new file mode 100755 index 00000000..c146ee65 --- /dev/null +++ b/marker_experiments/downstream/smoke_test.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Everything about the downstream pipeline that can be checked without a GPU. + +The GPU leg (nanochat pretrain + CORE/bpb eval) is NOT exercised here -- it needs +torch and an H100, neither of which the development container has. What *is* +exercised is the whole tokenizer-side path, which is where the boundary tokenizers +differ from the tokenizers this harness has run before: + + 1. fresh-process load by dotted class path (the harness loads tokenizers in + subprocesses; `BPETokenizer.load` on a boundary model raises + KeyError: 'BoundaryScriptPretokenizer' unless the registering module is imported) + 2. the `pynanochat.Tokenizer` contract, including `write_token_bytes`, which + produces the byte-length table the bits-per-byte metric is computed against + 3. dense id space: contiguous [0, n), synthetic BOS at n, vocab n+1 + 4. the uint16 bound `pretokenize.py` asserts (vocab <= 65535) + 5. round-trip through the adapter on marker-, caps- and digit-heavy text + 6. matched vocabulary across arms -- an unmatched set silently changes parameter + count and token horizon between arms, so this is a hard failure, not a warning + +Run: uv run python marker_experiments/downstream/smoke_test.py + uv run python marker_experiments/downstream/smoke_test.py --tokenizer-dir +""" + +import json +import os +import subprocess +import sys +import tempfile + +import cyclopts + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +DEFAULT_DIR = os.path.join(REPO, "marker_experiments", "tokenizers") +PYNANOCHAT_SRC = os.path.join(REPO, "eval", "py-nanochat") +DOTTED_BPE = "marker_experiments.downstream.boundary_tokenizer.BoundaryBPETokenizer" +DOTTED_MINGRAM = "marker_experiments.downstream.boundary_tokenizer.BoundaryMinGramModel" + + +def dotted_for(filename): + """`--tokenizer-class` for a tokenizer file. Trainers serialise differently: loading + a MinGram model with BPETokenizer.load raises KeyError: 'merge_rules'.""" + return DOTTED_MINGRAM if "mingram" in filename else DOTTED_BPE + +# marker pairs, an elided space, title case, all caps, mixed case (left literal), +# a digit run, and a script change inside one word span. +SAMPLES = [ + "The NASA report, 2024, says WiFi use grew 41.5% (n=1,203).", + "one two three\n\nfour\tfive", + "latin кириллица mixed, 123 456.", + "a,b c , d -- e.f.g 'quoted' \"double\"", + "「日本語」と한국어 and العربية 42", +] + +app = cyclopts.App() +FAILS: list[str] = [] + + +CURRENT = [""] + + +def check(name, ok, detail=""): + print(f" [{'ok ' if ok else 'FAIL'}] {name}" + (f" {detail}" if detail else ""), flush=True) + if not ok: + FAILS.append(f"{CURRENT[0]}: {name}") + + +def _import_adapter(): + """pynanochat is an optional extra; fall back to its source tree so this runs + in a container that has not done `uv sync --extra downstream`.""" + try: + from pynanochat.tokenizer_adapter import ScriptBPETokenizerAdapter + from pynanochat.tokenizer import write_token_bytes + except ModuleNotFoundError: + sys.path.insert(0, PYNANOCHAT_SRC) + from pynanochat.tokenizer_adapter import ScriptBPETokenizerAdapter + from pynanochat.tokenizer import write_token_bytes + return ScriptBPETokenizerAdapter, write_token_bytes + + +def fresh_process_load(path, dotted): + """Load in a subprocess with cwd outside the repo: exactly what the harness does.""" + code = ( + "import importlib,sys\n" + f"m,_,c={dotted!r}.rpartition('.')\n" + "t=getattr(importlib.import_module(m),c).load(sys.argv[1])\n" + "print(len(t.tokens))\n" + ) + r = subprocess.run( + # cwd is outside the repo, so the tokenizer path must be absolute. + [sys.executable, "-c", code, os.path.abspath(path)], cwd=tempfile.gettempdir(), + capture_output=True, text=True, + ) + if r.returncode != 0: + return None, r.stderr.strip().splitlines()[-1] if r.stderr else "no stderr" + return int(r.stdout.strip()), "" + + +@app.default +def main(tokenizer_dir: str = DEFAULT_DIR, pattern: str = "en_") -> None: + """Check the tokenizer-side downstream path for every matching tokenizer. + + Args: + tokenizer_dir: Directory of .json.gz tokenizers to check. + pattern: Only check files containing this substring. + """ + ScriptBPETokenizerAdapter, write_token_bytes = _import_adapter() + paths = sorted( + os.path.join(tokenizer_dir, f) + for f in os.listdir(tokenizer_dir) + if f.endswith(".json.gz") and pattern in f + ) + if not paths: + raise SystemExit(f"no tokenizers matching {pattern!r} in {tokenizer_dir}") + + long_text = "" + tsw = os.path.join(REPO, "tests", "data", "taylorswift.txt") + if os.path.exists(tsw): + long_text = open(tsw, encoding="utf-8").read() + + vocabs, token_counts = {}, {} + for path in paths: + name = os.path.basename(path).replace(".json.gz", "") + CURRENT[0] = name + print(f"\n{name}") + dotted = dotted_for(name) + + n_tokens, err = fresh_process_load(path, dotted) + check("fresh-process load by dotted path", n_tokens is not None, err) + if n_tokens is None: + continue + + import importlib + + mod, _, cls = dotted.rpartition(".") + tokenizer = getattr(importlib.import_module(mod), cls).load(path) + adapter = ScriptBPETokenizerAdapter(tokenizer) + vocab = adapter.get_vocab_size() + bos = adapter.get_bos_token_id() + vocabs[name] = vocab + + check("vocab == n_tokens + 1", vocab == n_tokens + 1, f"{vocab:,} vs {n_tokens:,}+1") + check("bos is the last dense id", bos == vocab - 1, f"bos={bos}") + check("uint16 bound (pretokenize.py asserts this)", vocab <= 65535, f"vocab={vocab:,}") + + for attr in ("encode", "decode", "get_bos_token_id", "get_vocab_size", + "get_special_tokens", "id_to_token"): + if not hasattr(adapter, attr): + check(f"contract: {attr}", False) + check("contract: all six methods present", True) + + bad = [] + for s in SAMPLES: + ids = adapter.encode(s, prepend=bos) + if ids[0] != bos or adapter.decode(ids) != s: + bad.append(s) + check("round-trip through adapter (5 samples)", not bad, f"failed: {bad[:1]}") + + if long_text: + ids = adapter.encode(long_text) + token_counts[name] = len(ids) + check( + "round-trip on taylorswift.txt", + adapter.decode(ids) == long_text, + f"{len(long_text):,} chars -> {len(ids):,} tokens " + f"({len(long_text) / len(ids):.4f} ch/tok)", + ) + in_range = all(0 <= i < vocab for i in ids) + check("all ids inside dense [0, vocab)", in_range) + + # batch encode is what pretokenize.py calls; it must agree with single encode. + batch = adapter.encode(SAMPLES[:3], prepend=bos) + singles = [adapter.encode(s, prepend=bos) for s in SAMPLES[:3]] + check("batch encode == single encode", [list(x) for x in batch] == [list(x) for x in singles]) + + # write_token_bytes serialises with torch.save, so it only runs where the + # downstream extra is installed. On a cluster this is the first real check. + try: + import torch # noqa: F401 + except ModuleNotFoundError: + print(" [skip] write_token_bytes (bpb byte table) torch not installed") + else: + with tempfile.TemporaryDirectory() as d: + write_token_bytes(adapter, d) + check("write_token_bytes (bpb byte table)", bool(os.listdir(d)), + ", ".join(os.listdir(d))) + + print("\nvocabulary sizes") + for name, v in sorted(vocabs.items()): + print(f" {name:<40} {v:,}") + if len(set(vocabs.values())) > 1: + print(" NOTE: these are the compression-grid tokenizers, which match " + "additional_vocab_size, not total vocab.\n" + " Use train_matched.py before running the downstream comparison.") + + if token_counts: + print("\ntokens on taylorswift.txt (lower is better)") + base = next((v for k, v in token_counts.items() if "plain" in k), None) + for name, n in sorted(token_counts.items(), key=lambda kv: kv[1]): + delta = f" {100 * (base - n) / base:+.2f}%" if base else "" + print(f" {name:<40} {n:,}{delta}") + + print(f"\n{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) + print("NOT covered here: nanochat pretrain + CORE/bpb eval (needs torch + a GPU).") + sys.exit(1 if FAILS else 0) + + +if __name__ == "__main__": + app() diff --git a/marker_experiments/downstream/train_matched.py b/marker_experiments/downstream/train_matched.py new file mode 100755 index 00000000..de4ea8bf --- /dev/null +++ b/marker_experiments/downstream/train_matched.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Train vocabulary-matched tokenizers for the downstream LM comparison. + +Why this exists instead of reusing `marker_experiments/tokenizers/*` +------------------------------------------------------------------- +The compression grid fixed `additional_vocab_size=32768` for every arm, so the arms +end up with *different* total vocabularies, because the boundary marker and the caps +codes are extra atomic tokens: + + plain 1710 atomic + 32768 = 34478 + bnd_wpd 1711 atomic + 32768 = 34479 + bnd_wpd_caps 1713 atomic + 32768 = 34481 + +That is the right control for a compression measurement -- each arm gets the same +number of *learned* merges -- but it is the wrong control downstream, where the +vocabulary size sets the embedding and unembedding shapes, hence the parameter count, +hence nanochat's compute-optimal token horizon. Downstream we match the *total*, and +let the learned budget absorb the difference: + + additional_vocab_size = total_vocab - len(pretokenizer.atomic_tokens) + +The default total is 34,685, which is what the MinGram downstream table used, so these +runs sit on the same axis as that table. + +Corpus +------ +Default `fineweb_en_5gb`, the corpus MinGram's downstream tokenizers were trained on, +and web text like the ClimbMix corpus the LM trains on. `finewiki_en_1gb` reproduces +the domain of our compression grid instead; expect slightly different absolute +chars/token from the numbers in the paper, because the grid withheld 500 evaluation +documents from training and the registry corpus does not. + +Usage +----- + uv run python marker_experiments/downstream/train_matched.py \ + --arms plain,bnd_wpd,bnd_wpd_caps --trainer bpe --workers 90 + + uv run python marker_experiments/downstream/train_matched.py \ + --arms plain,bnd_wpd --trainer mingram --workers 90 +""" + +import json +import os +import time + +import cyclopts + +from script_bpe.corpus.registry import load_corpus_by_name +from script_bpe.pretokenize import get_pretokenizer +from script_bpe.tokenizers.bpe.trainer import BPETrainer, BPETrainerConfig +from script_bpe.tokenizers.mingram.trainer import MinGramTrainer, MinGramTrainerConfig + +from marker_experiments.boundary_pretokenizer import ALL_VARIANTS, get_boundary_pretokenizer + +HERE = os.path.dirname(os.path.abspath(__file__)) +OUT_DIR = os.path.join(HERE, "tokenizers") +MANIFEST = os.path.join(HERE, "manifest.json") + +# 34,685 is the matched vocabulary of the MinGram downstream table. +DEFAULT_TOTAL_VOCAB = 34_685 +# f=1.15 is the repo default and the setting the overshoot sweep settled on. +DEFAULT_OVERSHOOT = 1.15 + +app = cyclopts.App() + + +def make_pretokenizer(arm: str): + """`plain` is the stock SCRIPT-v3 baseline; everything else is a boundary variant.""" + if arm == "plain": + return get_pretokenizer("scriptenc3_cb") + return get_boundary_pretokenizer(arm) + + +def build_corpus(corpus_name, pt, base_dir, text_file): + """Registry corpus, or -- with `text_file` -- a throwaway corpus from one local file. + + The local-file path exists so the vocabulary-matching logic and the full + train -> save -> load -> adapter chain can be exercised in seconds, before + committing to a multi-GB download. It is not a research setting: at this size the + vocabulary cannot be filled, so `total_vocab` is not reached and the matched-size + assertion is skipped. + """ + if not text_file: + return load_corpus_by_name(corpus_name, pt, base_dir=base_dir) + from script_bpe.corpus.base import PretokenizedCorpus + from script_bpe.corpus.registry import normalize_whitespace + + with open(text_file, encoding="utf-8") as f: + text = normalize_whitespace(f.read()) + return PretokenizedCorpus.from_text_batches( + name=corpus_name, + base_path=base_dir or os.path.join(HERE, "corpora"), + pretokenizer=pt, + text_batches=iter([[text]]), + num_workers=1, + ) + + +def train_one(arm, trainer_name, corpus_name, total_vocab, workers, overshoot, base_dir, text_file): + pt = make_pretokenizer(arm) + additional = total_vocab - len(pt.atomic_tokens) + assert additional > 0, f"{arm}: total_vocab {total_vocab} below {len(pt.atomic_tokens)} atomic tokens" + + corpus = build_corpus(corpus_name, pt, base_dir, text_file) + + t = time.time() + if trainer_name == "bpe": + cfg = BPETrainerConfig(additional_vocab_size=additional, num_workers=workers) + tokenizer = BPETrainer(pt, corpus, cfg).train() + elif trainer_name == "mingram": + cfg = MinGramTrainerConfig( + additional_vocab_size=additional, num_workers=workers, overshoot_factor=overshoot + ) + tokenizer = MinGramTrainer(pt, corpus, cfg).train() + else: + raise ValueError(f"unknown trainer {trainer_name!r}") + seconds = time.time() - t + + return tokenizer, { + "arm": arm, + "trainer": trainer_name, + "corpus": corpus_name, + "atomic_vocab": len(pt.atomic_tokens), + "additional_vocab_size": additional, + "total_vocab": len(tokenizer.tokens), + "train_seconds": round(seconds), + "overshoot_factor": overshoot if trainer_name == "mingram" else None, + } + + +def eval_compression(tokenizer, eval_path): + """chars/token and round-trip failures on the compression grid's held-out slice.""" + if not os.path.exists(eval_path): + return {} + with open(eval_path) as f: + texts = json.load(f) + chars = toks = fails = 0 + for text in texts: + ids = tokenizer.encode(text) + chars += len(text) + toks += len(ids) + if tokenizer.decode(ids) != text: + fails += 1 + return { + "eval_chars": chars, + "eval_tokens": toks, + "eval_chars_per_token": chars / toks, + "roundtrip_failures": fails, + } + + +@app.default +def main( + arms: str = "plain,bnd_wpd,bnd_wpd_caps", + trainer: str = "bpe", + corpus: str = "fineweb_en_5gb", + total_vocab: int = DEFAULT_TOTAL_VOCAB, + workers: int = 8, + overshoot: float = DEFAULT_OVERSHOOT, + out_dir: str = OUT_DIR, + corpus_base_dir: str | None = None, + eval_texts: str | None = None, + text_file: str | None = None, + force: bool = False, +) -> None: + """Train one vocabulary-matched tokenizer per arm and record a manifest. + + Args: + arms: Comma-separated arms. `plain` is the SCRIPT-v3 baseline; the rest are + boundary variants (bnd_w, bnd_wp, bnd_wpd and their _caps forms). + trainer: `bpe` or `mingram`. + corpus: Registry corpus name for tokenizer training (fineweb_en_5gb, finewiki_en_1gb, ...). + total_vocab: Matched total vocabulary. Every arm ends at exactly this size. + workers: Trainer worker processes. Set this to roughly the core count. + overshoot: MinGram BPE-init overshoot factor (ignored for `bpe`). + out_dir: Where the .json.gz tokenizers land. + corpus_base_dir: Pretokenized-corpus cache dir (defaults to the repo's). + eval_texts: Optional JSON list of held-out texts for a chars/token sanity check. + `marker_experiments/eval_texts/en.json` is the compression grid's slice. + text_file: Train on this local text file instead of a registry corpus. Seconds + rather than hours; for pipeline checks only, and vocabulary matching is not + enforced because the vocabulary cannot be filled at that size. + force: Retrain arms whose output file already exists. + """ + os.makedirs(out_dir, exist_ok=True) + manifest = json.load(open(MANIFEST)) if os.path.exists(MANIFEST) else {} + if text_file: + corpus = f"tiny_{os.path.basename(text_file).split('.')[0]}" + + trained = [] + for arm in [a.strip() for a in arms.split(",") if a.strip()]: + assert arm == "plain" or arm in ALL_VARIANTS, f"unknown arm {arm!r}" + key = f"{corpus}_{arm}_{trainer}_v{total_vocab}" + path = os.path.join(out_dir, f"{key}.json.gz") + if os.path.exists(path) and not force: + print(f"[train] {key}: exists, skipping", flush=True) + continue + + tokenizer, info = train_one( + arm, trainer, corpus, total_vocab, workers, overshoot, corpus_base_dir, text_file + ) + trained.append(key) + tokenizer.save(path) + info["path"] = os.path.relpath(path, os.path.dirname(HERE)) + if eval_texts: + info.update(eval_compression(tokenizer, eval_texts)) + manifest[key] = info + with open(MANIFEST, "w") as f: + json.dump(manifest, f, indent=2, sort_keys=True) + cpt = info.get("eval_chars_per_token") + print( + f"[train] {key}: vocab={info['total_vocab']:,} " + f"(atomic {info['atomic_vocab']} + {info['additional_vocab_size']:,}) " + f"{info['train_seconds']}s" + + (f" ch/tok={cpt:.4f} rt_fail={info['roundtrip_failures']}" if cpt else ""), + flush=True, + ) + + # A mismatch here silently biases the downstream comparison, so fail loudly. + sizes = { + k: v["total_vocab"] + for k, v in manifest.items() + if v["trainer"] == trainer and v["corpus"] == corpus + } + if text_file and len(set(sizes.values())) > 1: + # A tiny corpus may not contain enough distinct pairs to fill the vocabulary, + # in which case the arms legitimately stop at different sizes. Report, don't fail. + print(f"[train] tiny corpus did not fill the vocabulary in every arm: {sizes}") + elif len(set(sizes.values())) > 1: + raise SystemExit(f"vocabulary sizes are NOT matched across arms: {sizes}") + else: + print(f"[train] {len(sizes)} {trainer} arms on {corpus}, " + f"all at vocab {next(iter(sizes.values()), None)}") + + +if __name__ == "__main__": + app() diff --git a/marker_experiments/finewiki1gb_grid.py b/marker_experiments/finewiki1gb_grid.py new file mode 100644 index 00000000..acd9034f --- /dev/null +++ b/marker_experiments/finewiki1gb_grid.py @@ -0,0 +1,388 @@ +"""Boundary variants on FineWiki at 1 GB per language, 6 languages, BPE and MinGram. + +Grid +---- + pretokenizers : scriptenc3_cb (baseline), bnd_w, bnd_wp, bnd_wpd + languages : en, de, fi, ru, ar, ko (FINEWIKI_HYBRID6_CORPORA set) + trainers : bpe, mingram + vocabulary : 32,768 additional (paper_utils/unigram ADDITIONAL_VOCAB_SIZE) + => 48 cells + +All four pretokenizers are ScriptEncodingV3 with enforce_char_boundaries=True and +differ only in which unit kinds carry a boundary marker, so the comparison isolates +the boundary scheme. + +Cost and durability +------------------- +This is a long run: roughly 15-20 hours, dominated by MinGram at 1 GB. The +container's disk allowance is limited and has wiped the working tree three times +in this investigation, so: + + * every cell is committed AND pushed as soon as it finishes, trained tokenizer + included, so progress is never lost to a wipe or a reclaim; + * completed cells are skipped on restart, so re-running resumes; + * BPE runs for ALL languages before MinGram starts, so an interrupted run still + yields a complete picture for one trainer rather than half the languages for + both; + * text batches and pretokenized corpora stay untracked (too large to commit) + and are rebuilt if lost. + +Data is read via direct parquet row-group range requests. FineWiki has hundreds +of language configs and load_dataset(name=...) config resolution times out on a +cold cache here, while row-group reads sustain ~5M chars/s. normalize_whitespace +is applied exactly as the registry's finewiki loader does. +""" + +import gc +import json +import math +import os +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +import fsspec +import pyarrow.parquet as pq + +from script_bpe.pretokenize import get_pretokenizer +from script_bpe.corpus.base import PretokenizedCorpus +from script_bpe.corpus.registry import normalize_whitespace +from script_bpe.tokenizers.bpe.trainer import BPETrainer, BPETrainerConfig +from script_bpe.tokenizers.bpe.tokenizer import BPETokenizer +from script_bpe.tokenizers.mingram.trainer import MinGramTrainer, MinGramTrainerConfig +from script_bpe.tokenizers.unigram.model import UnigramToken +from script_bpe.utils import token_array + +from boundary_pretokenizer import BOUNDARY_VARIANTS, get_boundary_pretokenizer + +LANGS = ["en", "de", "fi", "ru", "ar", "ko"] +CHARS_PER_LANG = 1_000_000_000 +BLOCK_CHARS = 10_000_000 # text batch size on disk, matches registry's FINEWEB_BLOCK_MAX_CHARS +EVAL_DOCS = 500 +VOCAB = 32_768 +OVERSHOOT = 1.15 +NUM_WORKERS = 4 + +CORPORA = os.path.join(HERE, "corpora") +EVAL_DIR = os.path.join(HERE, "eval_texts") +TOKENIZERS = os.path.join(HERE, "tokenizers") +RESULT_PATH = os.path.join(HERE, "finewiki1gb_result.json") +RESOLVE = "https://huggingface.co/datasets/HuggingFaceFW/finewiki/resolve/main/{path}" +TREE_API = "https://huggingface.co/api/datasets/HuggingFaceFW/finewiki/tree/main/data/{lang}wiki" + +PRETOKENIZERS = {"plain": lambda: get_pretokenizer("scriptenc3_cb")} +PRETOKENIZERS.update({n: (lambda n=n: get_boundary_pretokenizer(n)) for n in BOUNDARY_VARIANTS}) + + +def log(msg): + print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + + +def git(*args, check=False): + return subprocess.run(["git", *args], cwd=os.path.dirname(HERE), capture_output=True, text=True, check=check) + + +def commit_cell(key): + """Persist a finished cell immediately; a wipe or reclaim must not cost work.""" + git("add", "marker_experiments") + if not git("diff", "--cached", "--quiet").returncode: + return # nothing staged + msg = ( + f"FineWiki 1GB grid: {key}\n\n" + "Auto-committed per cell so partial progress survives container wipes.\n\n" + "Co-Authored-By: Claude Opus 5 \n" + "Claude-Session: https://claude.ai/code/session_01PE4L4w3oY91vHCMK9uw32k\n" + ) + git("commit", "-q", "-m", msg) + for delay in (2, 4, 8, 16, 0): + if git("push", "-q", "-u", "origin", "claude/fineweb-space-neighbors-k10ufw").returncode == 0: + return + if delay: + time.sleep(delay) + log(f"WARNING: push failed for {key}; commit is local only") + + +def _lang_shards(lang): + """All parquet shards for a language, in order. + + Reading only shard 0 silently short-changes languages whose first shard holds + fewer than CHARS_PER_LANG characters: Arabic has 4 shards and shard 0 yields + 483M chars, Korean has 2 and shard 0 yields ~734M. English, German, Finnish + and Russian were unaffected because their first shard already exceeds 1 GB. + """ + import json as _json + import urllib.request + + for attempt in range(5): + try: + with urllib.request.urlopen(TREE_API.format(lang=lang), timeout=60) as r: + tree = _json.load(r) + shards = sorted(f["path"] for f in tree if f["path"].endswith(".parquet")) + if shards: + return shards + except Exception: + time.sleep(2 ** attempt) + raise RuntimeError(f"could not list shards for {lang}") + + +def _open_parquet(path, attempts=6): + last = None + for attempt in range(attempts): + try: + return pq.ParquetFile(fsspec.open(RESOLVE.format(path=path)).open()) + except Exception as e: # transient CDN/network failure + last = e + time.sleep(2 ** attempt) + raise RuntimeError(f"could not open {path}") from last + + +def stream_batches(lang): + """Yield ~BLOCK_CHARS batches of normalized text straight from parquet row groups. + + Nothing is staged on disk: writing 1 GB of text blocks per language is what + exhausted the session's disk allowance and wiped the working tree. Re-reading + per pretokenizer costs ~186s and keeps peak disk to the corpora alone. + """ + cur, cur_chars, total = [], 0, 0 + for shard in _lang_shards(lang): + if total >= CHARS_PER_LANG: + break + pf = _open_parquet(shard) + rg = 0 + while rg < pf.num_row_groups: + # The HF CDN returns transient 503s on long reads; a bare read killed a run + # partway through English. Retry the row group, reopening between attempts. + table = None + for attempt in range(6): + try: + table = pf.read_row_group(rg, columns=["text"]) + break + except Exception as e: + wait = 2 ** attempt + log(f"[{lang}] {shard} rg{rg} failed ({type(e).__name__}), retry in {wait}s") + time.sleep(wait) + pf = _open_parquet(shard) + if table is None: + raise RuntimeError(f"[{lang}] {shard} row group {rg} unreadable after retries") + rg += 1 + for x in table.column("text").to_pylist(): + if not x: + continue + x = normalize_whitespace(x) + if not x: + continue + cur.append(x) + cur_chars += len(x) + total += len(x) + if cur_chars >= BLOCK_CHARS: + yield cur + cur, cur_chars = [], 0 + if total >= CHARS_PER_LANG: + break + if cur: + yield cur + + +def train_batches(lang): + """stream_batches with the final EVAL_DOCS documents withheld. + + ensure_eval() takes the last EVAL_DOCS documents of the stream as the held-out slice, + but stream_batches yields the whole stream, so feeding it straight to the corpus + trains on the evaluation documents. They are ~1.3% of the corpus and the leak is + identical for every pretokenizer, so reported gaps are unaffected -- but absolute + chars/token is optimistic, so withhold them. + """ + tail = [] + for batch in stream_batches(lang): + tail.extend(batch) + if len(tail) > EVAL_DOCS: + emit = tail[:-EVAL_DOCS] + del tail[:-EVAL_DOCS] + if emit: + yield emit + + +def ensure_eval(lang): + """Held-out slice: the last EVAL_DOCS documents of the same 1 GB stream.""" + eval_path = os.path.join(EVAL_DIR, f"{lang}.json") + if os.path.exists(eval_path): + return json.load(open(eval_path)) + os.makedirs(EVAL_DIR, exist_ok=True) + t = time.time() + tail, total = [], 0 + for batch in stream_batches(lang): + total += sum(map(len, batch)) + tail.extend(batch) + del tail[:-EVAL_DOCS] + log(f"[{lang}] {total:,} chars streamed, eval slice {len(tail)} docs, {time.time()-t:.0f}s") + json.dump(tail, open(eval_path, "w")) + return tail + + +def drop_corpora(lang): + """Free a language's corpora once all its cells are done; peak disk is what wipes us.""" + import shutil + for tag in PRETOKENIZERS: + d = os.path.join(CORPORA, f"fw1gb_{lang}_{tag}") + if os.path.isdir(d): + shutil.rmtree(d, ignore_errors=True) + log(f"[{lang}] corpora removed") + + +class CachedInitMinGramTrainer(MinGramTrainer): + cache_tag = "unknown" + + def _build_bpe_init_tokens(self): + size = int(self.config.additional_vocab_size * self.config.overshoot_factor) + path = os.path.join(HERE, "bpe_init_cache", f"{self.cache_tag}_{size}.json.gz") + if os.path.exists(path): + self.logger.info(f"Reusing cached BPE init {path}") + m = BPETokenizer.load(path) + m.pretokenizer = self.pretokenizer + else: + cfg = BPETrainerConfig(additional_vocab_size=size, num_workers=self.config.num_workers) + m = BPETrainer(self.pretokenizer, self.corpus, cfg).train() + os.makedirs(os.path.dirname(path), exist_ok=True) + m.save(path) + tot = sum(max(1, t.current_count) for t in m.tokens.values()) + return [ + UnigramToken(id=t.id, atomic_tokens=token_array(t.atomic_tokens), + log_prob=math.log(max(1, t.current_count) / tot), + required=len(t.atomic_tokens) == 1) + for t in m.tokens.values() + ] + + +def analyse_vocab(tokenizer, pt): + marker_id = getattr(pt, "marker_token_id", None) + words, variants, ws_only = set(), {}, 0 + by_text = {} + for t in tokenizer.tokens.values(): + ids = list(t.atomic_tokens) + by_text.setdefault(pt.decode(ids), []).append(t) + core = [x for x in ids if x != marker_id] if marker_id is not None else ids + if not core: + continue + txt = pt.try_decode_strict(core) + if txt is None: + continue + if txt and txt.strip() == "": + ws_only += 1 + elif marker_id is not None: + if txt.isalpha() and ids[0] == marker_id and ids[-1] == marker_id and ids.count(marker_id) == 2: + words.add(txt) + elif txt and not any(c.isalnum() or c.isspace() for c in txt): + key = ("<|>" if ids[0] == marker_id else "") + txt + ("<|>" if ids[-1] == marker_id else "") + variants.setdefault(txt, set()).add(key) + else: + bare = txt[1:] if txt.startswith(" ") else txt + if bare.isalpha(): + words.add(bare) + dups = [x for x in by_text if x.startswith(" ") and len(x) > 1 and x[1:] in by_text] + dup_slots = sum(len(by_text[x]) + len(by_text[x[1:]]) for x in dups) + return { + "distinct_alpha_words_with_own_token": len(words), + "space_dup_pairs": len(dups), + "space_dup_vocab_frac": dup_slots / len(tokenizer.tokens), + "marker_variant_extra_slots": sum(len(v) - 1 for v in variants.values()), + "whitespace_only_vocab_entries": ws_only, + } + + +def main(): + os.makedirs(TOKENIZERS, exist_ok=True) + results = json.load(open(RESULT_PATH)) if os.path.exists(RESULT_PATH) else {} + + # Trainer-major: every BPE cell (~8 min each) finishes before any MinGram cell + # (~25-50 min each). The container wipes the working tree every ~30-60 min, and a + # cell longer than that interval can never complete, so the cheap trainer is run to + # completion across all six languages first. Corpora are still freed per language, + # and rebuilt for the MinGram pass -- ~300s, cheap next to the training it feeds. + for method in ["bpe", "mingram"]: + for lang in LANGS: + if all(f"{lang}_{tag}_{method}" in results for tag in PRETOKENIZERS): + log(f"{lang}/{method}: complete, skipping") + continue + eval_texts = ensure_eval(lang) + eval_chars = sum(map(len, eval_texts)) + for tag, make_pt in PRETOKENIZERS.items(): + key = f"{lang}_{tag}_{method}" + if key in results: + continue + try: + key = f"{lang}_{tag}_{method}" + if key in results: + continue + pt = make_pt() + corpus_name = f"fw1gb_{lang}_{tag}" + try: + corpus = PretokenizedCorpus(name=corpus_name, base_path=CORPORA, pretokenizer=pt) + except FileNotFoundError: + t = time.time() + corpus = PretokenizedCorpus.from_text_batches( + name=corpus_name, base_path=CORPORA, pretokenizer=pt, + text_batches=stream_batches(lang), num_workers=NUM_WORKERS, + ) + log(f"{lang}/{tag}: corpus built in {time.time()-t:.0f}s " + f"unique_chunks={corpus.metadata.get('unique_chunks'):,}") + + t = time.time() + if method == "bpe": + tokenizer = BPETrainer( + pt, corpus, BPETrainerConfig(additional_vocab_size=VOCAB, num_workers=NUM_WORKERS) + ).train() + else: + tr = CachedInitMinGramTrainer( + pt, corpus, + MinGramTrainerConfig(additional_vocab_size=VOCAB, num_workers=NUM_WORKERS, + overshoot_factor=OVERSHOOT), + ) + tr.cache_tag = f"{lang}_{tag}" + tokenizer = tr.train() + train_time = time.time() - t + + out = os.path.join(TOKENIZERS, f"{lang}_{tag}_{method}_{VOCAB//1024}k.json.gz") + tokenizer.save(out) + + toks = fails = 0 + for text in eval_texts: + ids = tokenizer.encode(text) + toks += len(ids) + if tokenizer.decode(ids) != text: + fails += 1 + + results[key] = { + "lang": lang, "pretokenizer": tag, "method": method, + "additional_vocab_size": VOCAB, "vocab_size": len(tokenizer.tokens), + "train_seconds": round(train_time), + "train_chars": corpus.metadata.get("atomic_tokens"), + "unique_chunks": corpus.metadata.get("unique_chunks"), + "eval_docs": len(eval_texts), "eval_chars": eval_chars, "eval_tokens": toks, + "eval_chars_per_token": eval_chars / toks, + "roundtrip_failures": fails, + "tokenizer_file": os.path.relpath(out, os.path.dirname(HERE)), + **analyse_vocab(tokenizer, pt), + } + with open(RESULT_PATH, "w") as f: + json.dump(results, f, indent=2) + log(f" {key}: {eval_chars/toks:.4f} ch/tok " + f"dup={results[key]['space_dup_pairs']} {round(train_time)}s rt={fails}") + del tokenizer + gc.collect() + commit_cell(key) + except Exception as e: + # One flaky cell must not abort the remaining grid; it is retried + # on the next run because it never entered results. + log(f" {key}: FAILED ({type(e).__name__}: {e}); continuing") + gc.collect() + + drop_corpora(lang) + + log(f"DONE: {len(results)} cells") + + +if __name__ == "__main__": + main() diff --git a/marker_experiments/finewiki1gb_result.json b/marker_experiments/finewiki1gb_result.json new file mode 100644 index 00000000..462a9c9a --- /dev/null +++ b/marker_experiments/finewiki1gb_result.json @@ -0,0 +1,527 @@ +{ + "en_plain_bpe": { + "lang": "en", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 152, + "train_chars": 2005507886, + "unique_chunks": 2350831, + "eval_docs": 500, + "eval_chars": 3602925, + "eval_tokens": 940477, + "eval_chars_per_token": 3.830954930317275, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/en_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 27729, + "space_dup_pairs": 3196, + "space_dup_vocab_frac": 0.18539358431463543, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 28 + }, + "en_bnd_w_bpe": { + "lang": "en", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 179, + "train_chars": 2079796128, + "unique_chunks": 2037466, + "eval_docs": 500, + "eval_chars": 3602925, + "eval_tokens": 1119274, + "eval_chars_per_token": 3.218983912786324, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/en_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 15493, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 1, + "whitespace_only_vocab_entries": 27 + }, + "en_bnd_wp_bpe": { + "lang": "en", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 157, + "train_chars": 2044360125, + "unique_chunks": 2041517, + "eval_docs": 500, + "eval_chars": 3602925, + "eval_tokens": 968560, + "eval_chars_per_token": 3.719877963161807, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/en_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 15381, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 220, + "whitespace_only_vocab_entries": 27 + }, + "en_bnd_wpd_bpe": { + "lang": "en", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 160, + "train_chars": 2036071982, + "unique_chunks": 2072665, + "eval_docs": 500, + "eval_chars": 3602925, + "eval_tokens": 906342, + "eval_chars_per_token": 3.9752378241326123, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/en_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 15196, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 213, + "whitespace_only_vocab_entries": 27 + }, + "de_plain_bpe": { + "lang": "de", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 329, + "train_chars": 2003480936, + "unique_chunks": 3685274, + "eval_docs": 500, + "eval_chars": 1404827, + "eval_tokens": 341721, + "eval_chars_per_token": 4.11103502564958, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/de_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 28061, + "space_dup_pairs": 3226, + "space_dup_vocab_frac": 0.18713382446777654, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 22 + }, + "de_bnd_w_bpe": { + "lang": "de", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 300, + "train_chars": 2068842794, + "unique_chunks": 3240417, + "eval_docs": 500, + "eval_chars": 1404827, + "eval_tokens": 390785, + "eval_chars_per_token": 3.5948846552452114, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/de_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11622, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 19 + }, + "de_bnd_wp_bpe": { + "lang": "de", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 300, + "train_chars": 2041300463, + "unique_chunks": 3244313, + "eval_docs": 500, + "eval_chars": 1404827, + "eval_tokens": 354449, + "eval_chars_per_token": 3.9634108150961067, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/de_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11547, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 186, + "whitespace_only_vocab_entries": 19 + }, + "de_bnd_wpd_bpe": { + "lang": "de", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 286, + "train_chars": 2032526795, + "unique_chunks": 3285362, + "eval_docs": 500, + "eval_chars": 1404827, + "eval_tokens": 336348, + "eval_chars_per_token": 4.176706863129853, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/de_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11375, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 188, + "whitespace_only_vocab_entries": 19 + }, + "fi_plain_bpe": { + "lang": "fi", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 353, + "train_chars": 1320380596, + "unique_chunks": 4004227, + "eval_docs": 500, + "eval_chars": 844063, + "eval_tokens": 211886, + "eval_chars_per_token": 3.983571354407559, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/fi_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 27999, + "space_dup_pairs": 3835, + "space_dup_vocab_frac": 0.22246069957654158, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 16 + }, + "fi_bnd_w_bpe": { + "lang": "fi", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 314, + "train_chars": 1363593520, + "unique_chunks": 3553098, + "eval_docs": 500, + "eval_chars": 844063, + "eval_tokens": 242552, + "eval_chars_per_token": 3.4799259540222303, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/fi_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11335, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 16 + }, + "fi_bnd_wp_bpe": { + "lang": "fi", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 325, + "train_chars": 1344362099, + "unique_chunks": 3555930, + "eval_docs": 500, + "eval_chars": 844063, + "eval_tokens": 218307, + "eval_chars_per_token": 3.8664037341908415, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/fi_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11286, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 154, + "whitespace_only_vocab_entries": 16 + }, + "fi_bnd_wpd_bpe": { + "lang": "fi", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 322, + "train_chars": 1339188108, + "unique_chunks": 3576886, + "eval_docs": 500, + "eval_chars": 844063, + "eval_tokens": 207687, + "eval_chars_per_token": 4.064110897648866, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/fi_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11001, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 146, + "whitespace_only_vocab_entries": 16 + }, + "ru_plain_bpe": { + "lang": "ru", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 241, + "train_chars": 2006038316, + "unique_chunks": 3681192, + "eval_docs": 500, + "eval_chars": 1894911, + "eval_tokens": 499253, + "eval_chars_per_token": 3.795492465743821, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ru_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 28341, + "space_dup_pairs": 3297, + "space_dup_vocab_frac": 0.19125239283021056, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 27 + }, + "ru_bnd_w_bpe": { + "lang": "ru", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 244, + "train_chars": 2082086384, + "unique_chunks": 3131931, + "eval_docs": 500, + "eval_chars": 1894911, + "eval_tokens": 578632, + "eval_chars_per_token": 3.2748119703023684, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ru_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 13161, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 1, + "whitespace_only_vocab_entries": 27 + }, + "ru_bnd_wp_bpe": { + "lang": "ru", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 235, + "train_chars": 2052149530, + "unique_chunks": 3136573, + "eval_docs": 500, + "eval_chars": 1894911, + "eval_tokens": 511278, + "eval_chars_per_token": 3.7062244023799185, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ru_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 13060, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 185, + "whitespace_only_vocab_entries": 27 + }, + "ru_bnd_wpd_bpe": { + "lang": "ru", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 231, + "train_chars": 2043543046, + "unique_chunks": 3170163, + "eval_docs": 500, + "eval_chars": 1894911, + "eval_tokens": 486271, + "eval_chars_per_token": 3.896820908505751, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ru_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 12761, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 185, + "whitespace_only_vocab_entries": 27 + }, + "ar_plain_bpe": { + "lang": "ar", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 206, + "train_chars": 2000092974, + "unique_chunks": 3219715, + "eval_docs": 500, + "eval_chars": 745245, + "eval_tokens": 187069, + "eval_chars_per_token": 3.983797422341489, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ar_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 27238, + "space_dup_pairs": 3159, + "space_dup_vocab_frac": 0.18324728812576135, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 29 + }, + "ar_bnd_w_bpe": { + "lang": "ar", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 189, + "train_chars": 2064090454, + "unique_chunks": 2879397, + "eval_docs": 500, + "eval_chars": 745245, + "eval_tokens": 213569, + "eval_chars_per_token": 3.4894811512906836, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ar_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 16202, + "space_dup_pairs": 7, + "space_dup_vocab_frac": 0.00040604425882421184, + "marker_variant_extra_slots": 12, + "whitespace_only_vocab_entries": 28 + }, + "ar_bnd_wp_bpe": { + "lang": "ar", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 200, + "train_chars": 2037071426, + "unique_chunks": 2884490, + "eval_docs": 500, + "eval_chars": 745245, + "eval_tokens": 192202, + "eval_chars_per_token": 3.8774050217999814, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ar_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 16093, + "space_dup_pairs": 7, + "space_dup_vocab_frac": 0.00040604425882421184, + "marker_variant_extra_slots": 209, + "whitespace_only_vocab_entries": 28 + }, + "ar_bnd_wpd_bpe": { + "lang": "ar", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 200, + "train_chars": 2027804463, + "unique_chunks": 2928610, + "eval_docs": 500, + "eval_chars": 745245, + "eval_tokens": 183565, + "eval_chars_per_token": 4.0598425625800125, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ar_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 15880, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.0003480379361350387, + "marker_variant_extra_slots": 198, + "whitespace_only_vocab_entries": 27 + }, + "ko_plain_bpe": { + "lang": "ko", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 456, + "train_chars": 1787998096, + "unique_chunks": 9474191, + "eval_docs": 500, + "eval_chars": 526821, + "eval_tokens": 236132, + "eval_chars_per_token": 2.2310445005335997, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ko_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 26153, + "space_dup_pairs": 5244, + "space_dup_vocab_frac": 0.3041939787690701, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 23 + }, + "ko_bnd_w_bpe": { + "lang": "ko", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 419, + "train_chars": 1911021760, + "unique_chunks": 8558506, + "eval_docs": 500, + "eval_chars": 526821, + "eval_tokens": 278479, + "eval_chars_per_token": 1.8917799905917503, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ko_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11993, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 22 + }, + "ko_bnd_wp_bpe": { + "lang": "ko", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 399, + "train_chars": 1869054034, + "unique_chunks": 8567034, + "eval_docs": 500, + "eval_chars": 526821, + "eval_tokens": 244643, + "eval_chars_per_token": 2.1534276476334906, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ko_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11895, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 225, + "whitespace_only_vocab_entries": 22 + }, + "ko_bnd_wpd_bpe": { + "lang": "ko", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 388, + "train_chars": 1858190672, + "unique_chunks": 8601093, + "eval_docs": 500, + "eval_chars": 526821, + "eval_tokens": 234063, + "eval_chars_per_token": 2.250765819458864, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/ko_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11760, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 218, + "whitespace_only_vocab_entries": 22 + }, + "en_plain_mingram": { + "lang": "en", + "pretokenizer": "plain", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 1579, + "train_chars": 2005507886, + "unique_chunks": 2350831, + "eval_docs": 500, + "eval_chars": 3602925, + "eval_tokens": 930064, + "eval_chars_per_token": 3.8738463159524508, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/en_plain_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 27534, + "space_dup_pairs": 3279, + "space_dup_vocab_frac": 0.1902082487383259, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 26 + } +} \ No newline at end of file diff --git a/marker_experiments/mingram250_grid.py b/marker_experiments/mingram250_grid.py new file mode 100644 index 00000000..2bf34acb --- /dev/null +++ b/marker_experiments/mingram250_grid.py @@ -0,0 +1,145 @@ +"""MinGram vs BPE for every boundary variant, at a scale that completes. + +The 1 GB grid's MinGram half needs ~26 min per cell, and the container clears the +working tree every ~30-60 min, so cells kept dying mid-flight: one of 24 landed. This +runs the same comparison at 250M characters, where a MinGram cell is ~8 min and a BPE +cell ~2 min, so a whole language finishes inside one window. + + languages : en, ru, ko (Latin, Cyrillic, Hangul -- one per script family) + pretokenizers : plain, bnd_w, bnd_wp, bnd_wpd + trainers : bpe, mingram (both, so the table is internally comparable) + vocabulary : 32,768 additional + => 24 cells + +BPE and MinGram share a language's corpora, and both trainers run before the corpora are +freed, so each corpus is built once. Evaluation documents are withheld from training via +train_batches. These numbers are NOT comparable to the 1 GB table in §4.1, which is a +different scale and trains on its own eval slice. +""" + +import gc +import json +import os +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +from script_bpe.corpus.base import PretokenizedCorpus +from script_bpe.pretokenize import get_pretokenizer +from script_bpe.tokenizers.bpe.trainer import BPETrainer, BPETrainerConfig +from script_bpe.tokenizers.mingram.trainer import MinGramTrainerConfig + +import finewiki1gb_grid as G +from boundary_pretokenizer import get_boundary_pretokenizer +from finewiki1gb_grid import ( + CORPORA, + NUM_WORKERS, + OVERSHOOT, + VOCAB, + CachedInitMinGramTrainer, + analyse_vocab, + commit_cell, + ensure_eval, + log, + train_batches, +) + +LANGS = ["en", "ru", "ko"] +CHARS = 250_000_000 +RESULT_PATH = os.path.join(HERE, "mingram250_result.json") +TOKENIZERS = os.path.join(HERE, "tokenizers") + +PRETOKENIZERS = {"plain": lambda: get_pretokenizer("scriptenc3_cb")} +PRETOKENIZERS.update({n: (lambda n=n: get_boundary_pretokenizer(n)) for n in ["bnd_w", "bnd_wp", "bnd_wpd"]}) + + +def main(): + os.makedirs(TOKENIZERS, exist_ok=True) + G.CHARS_PER_LANG = CHARS + results = json.load(open(RESULT_PATH)) if os.path.exists(RESULT_PATH) else {} + + for lang in LANGS: + if all(f"{lang}_{tag}_{m}" in results for tag in PRETOKENIZERS for m in ("bpe", "mingram")): + log(f"{lang}: complete, skipping") + continue + eval_texts = ensure_eval(lang) + eval_chars = sum(map(len, eval_texts)) + + for tag, make_pt in PRETOKENIZERS.items(): + pt = make_pt() + corpus_name = f"mg250_{lang}_{tag}" + corpus = None + for method in ["bpe", "mingram"]: + key = f"{lang}_{tag}_{method}" + if key in results: + continue + try: + if corpus is None: + try: + corpus = PretokenizedCorpus(name=corpus_name, base_path=CORPORA, pretokenizer=pt) + except FileNotFoundError: + t = time.time() + corpus = PretokenizedCorpus.from_text_batches( + name=corpus_name, base_path=CORPORA, pretokenizer=pt, + text_batches=train_batches(lang), num_workers=NUM_WORKERS, + ) + log(f"{lang}/{tag}: corpus built in {time.time()-t:.0f}s " + f"unique_chunks={corpus.metadata.get('unique_chunks'):,}") + + t = time.time() + if method == "bpe": + tokenizer = BPETrainer( + pt, corpus, BPETrainerConfig(additional_vocab_size=VOCAB, num_workers=NUM_WORKERS) + ).train() + else: + tr = CachedInitMinGramTrainer( + pt, corpus, + MinGramTrainerConfig(additional_vocab_size=VOCAB, num_workers=NUM_WORKERS, + overshoot_factor=OVERSHOOT), + ) + tr.cache_tag = f"mg250_{lang}_{tag}" + tokenizer = tr.train() + train_time = time.time() - t + + out = os.path.join(TOKENIZERS, f"mg250_{key}_32k.json.gz") + tokenizer.save(out) + + toks = fails = 0 + for text in eval_texts: + ids = tokenizer.encode(text) + toks += len(ids) + if tokenizer.decode(ids) != text: + fails += 1 + + results[key] = { + "lang": lang, "pretokenizer": tag, "method": method, + "additional_vocab_size": VOCAB, "vocab_size": len(tokenizer.tokens), + "train_seconds": round(train_time), + "unique_chunks": corpus.metadata.get("unique_chunks"), + "eval_docs": len(eval_texts), "eval_chars": eval_chars, "eval_tokens": toks, + "eval_chars_per_token": eval_chars / toks, + "roundtrip_failures": fails, + "tokenizer_file": os.path.relpath(out, os.path.dirname(HERE)), + **analyse_vocab(tokenizer, pt), + } + with open(RESULT_PATH, "w") as f: + json.dump(results, f, indent=2) + log(f" {key}: {eval_chars/toks:.4f} ch/tok {round(train_time)}s rt={fails}") + del tokenizer + gc.collect() + commit_cell(key) + except Exception as e: + log(f" {key}: FAILED ({type(e).__name__}: {e}); continuing") + gc.collect() + import shutil + for tag in PRETOKENIZERS: + shutil.rmtree(os.path.join(CORPORA, f"mg250_{lang}_{tag}"), ignore_errors=True) + log(f"[{lang}] corpora removed") + + log(f"DONE: {len(results)} cells") + + +if __name__ == "__main__": + main() diff --git a/marker_experiments/mingram250_result.json b/marker_experiments/mingram250_result.json new file mode 100644 index 00000000..39b5aa7e --- /dev/null +++ b/marker_experiments/mingram250_result.json @@ -0,0 +1,482 @@ +{ + "en_plain_bpe": { + "lang": "en", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 121, + "unique_chunks": 1020944, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 590550, + "eval_chars_per_token": 3.757990009313352, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 27667, + "space_dup_pairs": 3212, + "space_dup_vocab_frac": 0.18632171239631068, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 29 + }, + "en_plain_mingram": { + "lang": "en", + "pretokenizer": "plain", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34477, + "train_seconds": 826, + "unique_chunks": 1020944, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 583737, + "eval_chars_per_token": 3.8018508335089263, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_plain_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 27440, + "space_dup_pairs": 3256, + "space_dup_vocab_frac": 0.18887954288366157, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 30 + }, + "en_bnd_w_bpe": { + "lang": "en", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 110, + "unique_chunks": 891235, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 696707, + "eval_chars_per_token": 3.1853863962899758, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 15295, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 2, + "whitespace_only_vocab_entries": 28 + }, + "en_bnd_w_mingram": { + "lang": "en", + "pretokenizer": "bnd_w", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 843, + "unique_chunks": 891235, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 691938, + "eval_chars_per_token": 3.2073408311149265, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_bnd_w_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 17074, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 2, + "whitespace_only_vocab_entries": 29 + }, + "en_bnd_wp_bpe": { + "lang": "en", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 150, + "unique_chunks": 893113, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 611447, + "eval_chars_per_token": 3.6295557914259127, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 15182, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 222, + "whitespace_only_vocab_entries": 28 + }, + "en_bnd_wp_mingram": { + "lang": "en", + "pretokenizer": "bnd_wp", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 809, + "unique_chunks": 893113, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 606682, + "eval_chars_per_token": 3.6580630379671724, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_bnd_wp_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 16996, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 189, + "whitespace_only_vocab_entries": 29 + }, + "en_bnd_wpd_bpe": { + "lang": "en", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 107, + "unique_chunks": 906491, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 571265, + "eval_chars_per_token": 3.884853789397215, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 15021, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 212, + "whitespace_only_vocab_entries": 28 + }, + "en_bnd_wpd_mingram": { + "lang": "en", + "pretokenizer": "bnd_wpd", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 794, + "unique_chunks": 906491, + "eval_docs": 500, + "eval_chars": 2219281, + "eval_tokens": 566652, + "eval_chars_per_token": 3.9164796030014895, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_en_bnd_wpd_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 16853, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 171, + "whitespace_only_vocab_entries": 28 + }, + "ru_plain_bpe": { + "lang": "ru", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 151, + "unique_chunks": 1281612, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1300911, + "eval_chars_per_token": 3.8093797346628633, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 28442, + "space_dup_pairs": 3232, + "space_dup_vocab_frac": 0.18748187249840478, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 24 + }, + "ru_plain_mingram": { + "lang": "ru", + "pretokenizer": "plain", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34476, + "train_seconds": 489, + "unique_chunks": 1281612, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1282645, + "eval_chars_per_token": 3.8636286735612737, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_plain_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 28509, + "space_dup_pairs": 3077, + "space_dup_vocab_frac": 0.17850098619329388, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 22 + }, + "ru_bnd_w_bpe": { + "lang": "ru", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 132, + "unique_chunks": 1109145, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1472568, + "eval_chars_per_token": 3.365320990270059, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 13516, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 1, + "whitespace_only_vocab_entries": 22 + }, + "ru_bnd_w_mingram": { + "lang": "ru", + "pretokenizer": "bnd_w", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 527, + "unique_chunks": 1109145, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1457854, + "eval_chars_per_token": 3.3992868970418164, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_bnd_w_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 14954, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.00029004002552352224, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 22 + }, + "ru_bnd_wp_bpe": { + "lang": "ru", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 131, + "unique_chunks": 1110724, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1318690, + "eval_chars_per_token": 3.75802045969864, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 13415, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 176, + "whitespace_only_vocab_entries": 22 + }, + "ru_bnd_wp_mingram": { + "lang": "ru", + "pretokenizer": "bnd_wp", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 458, + "unique_chunks": 1110724, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1303963, + "eval_chars_per_token": 3.8004636634628435, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_bnd_wp_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 14887, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 149, + "whitespace_only_vocab_entries": 22 + }, + "ru_bnd_wpd_bpe": { + "lang": "ru", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 132, + "unique_chunks": 1125178, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1270532, + "eval_chars_per_token": 3.9004637427471325, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 13152, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.0002900316134458656, + "marker_variant_extra_slots": 174, + "whitespace_only_vocab_entries": 22 + }, + "ru_bnd_wpd_mingram": { + "lang": "ru", + "pretokenizer": "bnd_wpd", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 461, + "unique_chunks": 1125178, + "eval_docs": 500, + "eval_chars": 4955664, + "eval_tokens": 1254197, + "eval_chars_per_token": 3.9512644345346066, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ru_bnd_wpd_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 14597, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.00029004002552352224, + "marker_variant_extra_slots": 142, + "whitespace_only_vocab_entries": 22 + }, + "ko_plain_bpe": { + "lang": "ko", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 282, + "unique_chunks": 4337052, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 291951, + "eval_chars_per_token": 2.179345164085754, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_plain_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 26128, + "space_dup_pairs": 5266, + "space_dup_vocab_frac": 0.30547015488137363, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 26 + }, + "ko_plain_mingram": { + "lang": "ko", + "pretokenizer": "plain", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34478, + "train_seconds": 749, + "unique_chunks": 4337052, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 288487, + "eval_chars_per_token": 2.205513593333495, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_plain_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 26192, + "space_dup_pairs": 5134, + "space_dup_vocab_frac": 0.29781309820755264, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 23 + }, + "ko_bnd_w_bpe": { + "lang": "ko", + "pretokenizer": "bnd_w", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 247, + "unique_chunks": 3950662, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 351917, + "eval_chars_per_token": 1.8079888155445745, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_bnd_w_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11873, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 23 + }, + "ko_bnd_w_mingram": { + "lang": "ko", + "pretokenizer": "bnd_w", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 728, + "unique_chunks": 3950662, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 350900, + "eval_chars_per_token": 1.8132288401253918, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_bnd_w_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 12378, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 23 + }, + "ko_bnd_wp_bpe": { + "lang": "ko", + "pretokenizer": "bnd_wp", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 233, + "unique_chunks": 3955004, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 304832, + "eval_chars_per_token": 2.0872546189376444, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_bnd_wp_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11775, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 230, + "whitespace_only_vocab_entries": 23 + }, + "ko_bnd_wp_mingram": { + "lang": "ko", + "pretokenizer": "bnd_wp", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 696, + "unique_chunks": 3955004, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 303839, + "eval_chars_per_token": 2.09407613900783, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_bnd_wp_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 12290, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 205, + "whitespace_only_vocab_entries": 22 + }, + "ko_bnd_wpd_bpe": { + "lang": "ko", + "pretokenizer": "bnd_wpd", + "method": "bpe", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 245, + "unique_chunks": 3972102, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 293920, + "eval_chars_per_token": 2.164745508982036, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_bnd_wpd_bpe_32k.json.gz", + "distinct_alpha_words_with_own_token": 11668, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 218, + "whitespace_only_vocab_entries": 23 + }, + "ko_bnd_wpd_mingram": { + "lang": "ko", + "pretokenizer": "bnd_wpd", + "method": "mingram", + "additional_vocab_size": 32768, + "vocab_size": 34479, + "train_seconds": 709, + "unique_chunks": 3972102, + "eval_docs": 500, + "eval_chars": 636262, + "eval_tokens": 292904, + "eval_chars_per_token": 2.1722543905170295, + "roundtrip_failures": 0, + "tokenizer_file": "marker_experiments/tokenizers/mg250_ko_bnd_wpd_mingram_32k.json.gz", + "distinct_alpha_words_with_own_token": 12189, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023202529075669247, + "marker_variant_extra_slots": 191, + "whitespace_only_vocab_entries": 20 + } +} \ No newline at end of file diff --git a/marker_experiments/multilang_grid.py b/marker_experiments/multilang_grid.py new file mode 100644 index 00000000..6f499883 --- /dev/null +++ b/marker_experiments/multilang_grid.py @@ -0,0 +1,246 @@ +"""Boundary markers across the hybrid/ 6-language FineWiki set. + +Languages are the ones in FINEWIKI_HYBRID6_CORPORA (script_bpe/corpus/registry.py): +en, de, fi, ru, ar, ko -- Latin x3, Cyrillic, Arabic, Hangul. All six are in +DEFAULT_SCRIPTS_LM_WITH_SPACES, so all six are word-wrapped by the marker schemes. + +Deviations from the registry's finewiki_{lang}_1gb, and why: + * CHARS_PER_LANG below the registry's 1 GB cap, for compute budget. Everything + else is matched: same dataset, same normalize_whitespace transform the + registry applies to finewiki (which collapses [ \\t]+ to a single space, so + multi-space runs are absent by construction here). + * Data is read via direct parquet row-group range requests rather than + load_dataset(name=...). FineWiki has hundreds of language configs and config + resolution times out on a cold cache in this environment, while row-group + range reads run at ~5M chars/s. + +Three pretokenizers, all on ScriptEncodingV3 with enforce_char_boundaries: + plain : scriptenc3_cb, the current baseline + v4 : markers on word spans (unconditional) + punctuation (space side only) + v5 : v4 + digits markable (space side only) + +Results are written after every cell, and completed cells are skipped on re-run, +so this survives interruption. +""" + +import os +import sys +import json +import time +import math + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +import fsspec +import pyarrow.parquet as pq + +from script_bpe.pretokenize import get_pretokenizer +from script_bpe.pretokenize.scriptencoding import ScriptEncodingV3 +from script_bpe.corpus.base import PretokenizedCorpus +from script_bpe.corpus.registry import normalize_whitespace +from script_bpe.tokenizers.bpe.trainer import BPETrainer, BPETrainerConfig +from script_bpe.tokenizers.bpe.tokenizer import BPETokenizer +from script_bpe.tokenizers.mingram.trainer import MinGramTrainer, MinGramTrainerConfig +from script_bpe.tokenizers.unigram.model import UnigramToken +from script_bpe.utils import token_array + +from scriptenc_marker_v4 import MarkerV4Pretokenizer, MarkerV4PretokenizerConfig +from scriptenc_marker_v5 import MarkerV5Pretokenizer, MarkerV5PretokenizerConfig + +LANGS = ["en", "de", "fi", "ru", "ar", "ko"] +CHARS_PER_LANG = 100_000_000 +EVAL_DOCS = 500 +BPE_VOCABS = [16_000, 32_000, 64_000] +MINGRAM_VOCABS = [64_000] +MINGRAM_LANGS = ["en", "ru", "ko"] # one per script family, to check trainer-independence +OVERSHOOT = 1.15 +NUM_WORKERS = 4 + +CORPORA = os.path.join(HERE, "corpora") +EVAL_DIR = os.path.join(HERE, "eval_texts") +RESULT_PATH = os.path.join(HERE, "multilang_result.json") +URL = "https://huggingface.co/datasets/HuggingFaceFW/finewiki/resolve/main/data/{lang}wiki/000_00000.parquet" + +PRETOKENIZERS = { + "plain": lambda: get_pretokenizer("scriptenc3_cb"), + "v4": lambda: MarkerV4Pretokenizer(MarkerV4PretokenizerConfig(script_config=ScriptEncodingV3)), + "v5": lambda: MarkerV5Pretokenizer(MarkerV5PretokenizerConfig(script_config=ScriptEncodingV3)), +} + + +def read_lang(lang): + """Stream row groups until CHARS_PER_LANG, applying the registry's finewiki transform.""" + t = time.time() + f = fsspec.open(URL.format(lang=lang)).open() + pf = pq.ParquetFile(f) + texts, total = [], 0 + for rg in range(pf.num_row_groups): + for x in pf.read_row_group(rg, columns=["text"]).column("text").to_pylist(): + if not x: + continue + x = normalize_whitespace(x) + if not x: + continue + texts.append(x) + total += len(x) + if total >= CHARS_PER_LANG: + break + print(f"[{lang}] {len(texts):,} docs, {total:,} chars in {time.time()-t:.0f}s", flush=True) + return texts + + +class CachedInitMinGramTrainer(MinGramTrainer): + cache_tag = "unknown" + + def _build_bpe_init_tokens(self): + size = int(self.config.additional_vocab_size * self.config.overshoot_factor) + path = os.path.join(HERE, "bpe_init_cache", f"{self.cache_tag}_{size}.json.gz") + if os.path.exists(path): + self.logger.info(f"Reusing cached BPE init {path}") + m = BPETokenizer.load(path) + m.pretokenizer = self.pretokenizer + else: + cfg = BPETrainerConfig(additional_vocab_size=size, num_workers=self.config.num_workers) + m = BPETrainer(self.pretokenizer, self.corpus, cfg).train() + os.makedirs(os.path.dirname(path), exist_ok=True) + m.save(path) + tot = sum(max(1, t.current_count) for t in m.tokens.values()) + return [ + UnigramToken(id=t.id, atomic_tokens=token_array(t.atomic_tokens), + log_prob=math.log(max(1, t.current_count) / tot), + required=len(t.atomic_tokens) == 1) + for t in m.tokens.values() + ] + + +def analyse_vocab(tokenizer, pt): + """Duplicate-pair vocabulary tax, marker-variant cost, and word coverage.""" + marker_id = getattr(pt, "marker_token_id", None) + words, variants, ws_only = set(), {}, 0 + by_text = {} + for t in tokenizer.tokens.values(): + ids = list(t.atomic_tokens) + by_text.setdefault(pt.decode(ids), []).append(t) + core = [x for x in ids if x != marker_id] if marker_id is not None else ids + if not core: + continue + txt = pt.try_decode_strict(core) + if txt is None: + continue + if txt and txt.strip() == "": + ws_only += 1 + elif marker_id is not None: + if txt.isalpha() and ids[0] == marker_id and ids[-1] == marker_id and ids.count(marker_id) == 2: + words.add(txt) + elif txt and not any(c.isalnum() or c.isspace() for c in txt): + key = ("<|>" if ids[0] == marker_id else "") + txt + ("<|>" if ids[-1] == marker_id else "") + variants.setdefault(txt, set()).add(key) + else: + words.add(txt[1:] if txt.startswith(" ") else txt) if ( + (txt[1:] if txt.startswith(" ") else txt).isalpha()) else None + dups = [x for x in by_text if x.startswith(" ") and len(x) > 1 and x[1:] in by_text] + dup_slots = sum(len(by_text[x]) + len(by_text[x[1:]]) for x in dups) + return { + "distinct_alpha_words_with_own_token": len(words), + "space_dup_pairs": len(dups), + "space_dup_vocab_frac": dup_slots / len(tokenizer.tokens), + "marker_variant_extra_slots": sum(len(v) - 1 for v in variants.values()), + "whitespace_only_vocab_entries": ws_only, + } + + +def main(): + os.makedirs(EVAL_DIR, exist_ok=True) + results = json.load(open(RESULT_PATH)) if os.path.exists(RESULT_PATH) else {} + jobs = [("bpe", v) for v in BPE_VOCABS] + [("mingram", v) for v in MINGRAM_VOCABS] + + for lang in LANGS: + needed = [ + f"{lang}_{tag}_{m}_{v//1000}k" + for tag in PRETOKENIZERS for m, v in jobs + if not (m == "mingram" and lang not in MINGRAM_LANGS) + ] + if all(k in results for k in needed): + print(f"=== {lang}: all cells present, skipping ===", flush=True) + continue + + eval_path = os.path.join(EVAL_DIR, f"{lang}.json") + train_texts = None + if os.path.exists(eval_path): + eval_texts = json.load(open(eval_path)) + else: + texts = read_lang(lang) + eval_texts, train_texts = texts[-EVAL_DOCS:], texts[:-EVAL_DOCS] + json.dump(eval_texts, open(eval_path, "w")) + eval_chars = sum(map(len, eval_texts)) + + for tag, make_pt in PRETOKENIZERS.items(): + corpus_name = f"finewiki6_{lang}_{tag}" + pt = make_pt() + try: + corpus = PretokenizedCorpus(name=corpus_name, base_path=CORPORA, pretokenizer=pt) + except FileNotFoundError: + if train_texts is None: + texts = read_lang(lang) + eval_texts = texts[-EVAL_DOCS:] + train_texts = texts[:-EVAL_DOCS] + eval_chars = sum(map(len, eval_texts)) + t = time.time() + corpus = PretokenizedCorpus.from_texts( + name=corpus_name, base_path=CORPORA, pretokenizer=pt, + texts=train_texts, num_workers=NUM_WORKERS, + ) + print(f"[{lang}/{tag}] corpus built in {time.time()-t:.0f}s " + f"unique_chunks={corpus.metadata.get('unique_chunks'):,}", flush=True) + + for method, vocab in jobs: + if method == "mingram" and lang not in MINGRAM_LANGS: + continue + key = f"{lang}_{tag}_{method}_{vocab//1000}k" + if key in results: + continue + t = time.time() + if method == "bpe": + tokenizer = BPETrainer( + pt, corpus, BPETrainerConfig(additional_vocab_size=vocab, num_workers=NUM_WORKERS) + ).train() + else: + tr = CachedInitMinGramTrainer( + pt, corpus, + MinGramTrainerConfig(additional_vocab_size=vocab, num_workers=NUM_WORKERS, + overshoot_factor=OVERSHOOT), + ) + tr.cache_tag = f"{lang}_{tag}" + tokenizer = tr.train() + train_time = time.time() - t + + toks = fails = 0 + for text in eval_texts: + ids = tokenizer.encode(text) + toks += len(ids) + if tokenizer.decode(ids) != text: + fails += 1 + + results[key] = { + "lang": lang, "pretokenizer": tag, "method": method, + "additional_vocab_size": vocab, "vocab_size": len(tokenizer.tokens), + "train_seconds": round(train_time), + "unique_chunks": corpus.metadata.get("unique_chunks"), + "eval_docs": len(eval_texts), "eval_chars": eval_chars, "eval_tokens": toks, + "eval_chars_per_token": eval_chars / toks, + "roundtrip_failures": fails, + **analyse_vocab(tokenizer, pt), + } + print(f" {key}: {eval_chars/toks:.4f} ch/tok dup={results[key]['space_dup_pairs']} " + f"{round(train_time)}s rt={fails}", flush=True) + with open(RESULT_PATH, "w") as f: + json.dump(results, f, indent=2) + train_texts = None # free before the next language + + print("\n=== DONE ===") + print(f"{len(results)} cells in {RESULT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/marker_experiments/multilang_result.json b/marker_experiments/multilang_result.json new file mode 100644 index 00000000..9f43ff97 --- /dev/null +++ b/marker_experiments/multilang_result.json @@ -0,0 +1,1199 @@ +{ + "en_plain_bpe_16k": { + "lang": "en", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17710, + "train_seconds": 46, + "unique_chunks": 510925, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 288703, + "eval_chars_per_token": 3.5851549862661627, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 13749, + "space_dup_pairs": 1381, + "space_dup_vocab_frac": 0.155957086391869, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 20 + }, + "en_plain_bpe_32k": { + "lang": "en", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33710, + "train_seconds": 59, + "unique_chunks": 510925, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 269331, + "eval_chars_per_token": 3.843022154894906, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27028, + "space_dup_pairs": 3058, + "space_dup_vocab_frac": 0.18142984277662413, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 25 + }, + "en_plain_bpe_64k": { + "lang": "en", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 83, + "unique_chunks": 510925, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 255793, + "eval_chars_per_token": 4.0464164382919, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 54456, + "space_dup_pairs": 6767, + "space_dup_vocab_frac": 0.20596560645259473, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 42 + }, + "en_plain_mingram_64k": { + "lang": "en", + "pretokenizer": "plain", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65709, + "train_seconds": 382, + "unique_chunks": 510925, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 253581, + "eval_chars_per_token": 4.081713535320075, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 54204, + "space_dup_pairs": 6918, + "space_dup_vocab_frac": 0.21056476281787884, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 42 + }, + "en_v4_bpe_16k": { + "lang": "en", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 43, + "unique_chunks": 448640, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 296605, + "eval_chars_per_token": 3.4896411051735474, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 6506, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00045169668567556886, + "marker_variant_extra_slots": 138, + "whitespace_only_vocab_entries": 19 + }, + "en_v4_bpe_32k": { + "lang": "en", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 56, + "unique_chunks": 448640, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 275376, + "eval_chars_per_token": 3.758660885480216, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 14676, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 202, + "whitespace_only_vocab_entries": 25 + }, + "en_v4_bpe_64k": { + "lang": "en", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 76, + "unique_chunks": 448640, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 261090, + "eval_chars_per_token": 3.9643226473629785, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 33001, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.00018261782654350109, + "marker_variant_extra_slots": 304, + "whitespace_only_vocab_entries": 40 + }, + "en_v4_mingram_64k": { + "lang": "en", + "pretokenizer": "v4", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 368, + "unique_chunks": 448640, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 259517, + "eval_chars_per_token": 3.988351437478084, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 37661, + "space_dup_pairs": 9, + "space_dup_vocab_frac": 0.0002739267398152516, + "marker_variant_extra_slots": 275, + "whitespace_only_vocab_entries": 41 + }, + "en_v5_bpe_16k": { + "lang": "en", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 42, + "unique_chunks": 456044, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 284546, + "eval_chars_per_token": 3.6375313657545703, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 6299, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00045169668567556886, + "marker_variant_extra_slots": 131, + "whitespace_only_vocab_entries": 19 + }, + "en_v5_bpe_32k": { + "lang": "en", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 54, + "unique_chunks": 456044, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 262435, + "eval_chars_per_token": 3.944005182235601, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 14450, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 188, + "whitespace_only_vocab_entries": 25 + }, + "en_v5_bpe_64k": { + "lang": "en", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 79, + "unique_chunks": 456044, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 247948, + "eval_chars_per_token": 4.1744438349976605, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 31708, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.00018261782654350109, + "marker_variant_extra_slots": 285, + "whitespace_only_vocab_entries": 38 + }, + "en_v5_mingram_64k": { + "lang": "en", + "pretokenizer": "v5", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 379, + "unique_chunks": 456044, + "eval_docs": 500, + "eval_chars": 1035045, + "eval_tokens": 246466, + "eval_chars_per_token": 4.199544764795144, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 36276, + "space_dup_pairs": 8, + "space_dup_vocab_frac": 0.0002434904353913348, + "marker_variant_extra_slots": 246, + "whitespace_only_vocab_entries": 38 + }, + "de_plain_bpe_16k": { + "lang": "de", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17710, + "train_seconds": 80, + "unique_chunks": 810516, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 385331, + "eval_chars_per_token": 3.7237414067386223, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 13852, + "space_dup_pairs": 1459, + "space_dup_vocab_frac": 0.1647656691134952, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 14 + }, + "de_plain_bpe_32k": { + "lang": "de", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33710, + "train_seconds": 94, + "unique_chunks": 810516, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 355115, + "eval_chars_per_token": 4.040586852146488, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27537, + "space_dup_pairs": 3143, + "space_dup_vocab_frac": 0.18647285671907446, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 20 + }, + "de_plain_bpe_64k": { + "lang": "de", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 121, + "unique_chunks": 810516, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 333758, + "eval_chars_per_token": 4.299141893228027, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 54940, + "space_dup_pairs": 6677, + "space_dup_vocab_frac": 0.2032262973672196, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 31 + }, + "de_v4_bpe_16k": { + "lang": "de", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 70, + "unique_chunks": 729395, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 404146, + "eval_chars_per_token": 3.550382782459804, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4647, + "space_dup_pairs": 3, + "space_dup_vocab_frac": 0.0003387725142566766, + "marker_variant_extra_slots": 116, + "whitespace_only_vocab_entries": 14 + }, + "de_v4_bpe_32k": { + "lang": "de", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 90, + "unique_chunks": 729395, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 370027, + "eval_chars_per_token": 3.8777521640312735, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 11399, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 167, + "whitespace_only_vocab_entries": 20 + }, + "de_v4_bpe_64k": { + "lang": "de", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 113, + "unique_chunks": 729395, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 345732, + "eval_chars_per_token": 4.1502464336538125, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27062, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.00018261782654350109, + "marker_variant_extra_slots": 237, + "whitespace_only_vocab_entries": 28 + }, + "de_v5_bpe_16k": { + "lang": "de", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 72, + "unique_chunks": 738265, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 383690, + "eval_chars_per_token": 3.739667439860304, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4489, + "space_dup_pairs": 3, + "space_dup_vocab_frac": 0.0003387725142566766, + "marker_variant_extra_slots": 114, + "whitespace_only_vocab_entries": 14 + }, + "de_v5_bpe_32k": { + "lang": "de", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 89, + "unique_chunks": 738265, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 348500, + "eval_chars_per_token": 4.1172826398852225, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 11197, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 165, + "whitespace_only_vocab_entries": 20 + }, + "de_v5_bpe_64k": { + "lang": "de", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 115, + "unique_chunks": 738265, + "eval_docs": 500, + "eval_chars": 1434873, + "eval_tokens": 323582, + "eval_chars_per_token": 4.434341217991112, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 26665, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.00015218152211958423, + "marker_variant_extra_slots": 238, + "whitespace_only_vocab_entries": 27 + }, + "fi_plain_bpe_16k": { + "lang": "fi", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17710, + "train_seconds": 110, + "unique_chunks": 1200006, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 363537, + "eval_chars_per_token": 3.6566484291832744, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 13673, + "space_dup_pairs": 1782, + "space_dup_vocab_frac": 0.20124223602484473, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 13 + }, + "fi_plain_bpe_32k": { + "lang": "fi", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33710, + "train_seconds": 124, + "unique_chunks": 1200006, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 328957, + "eval_chars_per_token": 4.041035758472992, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27435, + "space_dup_pairs": 3672, + "space_dup_vocab_frac": 0.21785820231385344, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 18 + }, + "fi_plain_bpe_64k": { + "lang": "fi", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 159, + "unique_chunks": 1200006, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 302151, + "eval_chars_per_token": 4.399545260482341, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 54644, + "space_dup_pairs": 7634, + "space_dup_vocab_frac": 0.23235428397504185, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 23 + }, + "fi_v4_bpe_16k": { + "lang": "fi", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 109, + "unique_chunks": 1088693, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 373873, + "eval_chars_per_token": 3.555557635881703, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4218, + "space_dup_pairs": 3, + "space_dup_vocab_frac": 0.0003387725142566766, + "marker_variant_extra_slots": 100, + "whitespace_only_vocab_entries": 13 + }, + "fi_v4_bpe_32k": { + "lang": "fi", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 120, + "unique_chunks": 1088693, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 337521, + "eval_chars_per_token": 3.9385016043446184, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 10877, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 144, + "whitespace_only_vocab_entries": 18 + }, + "fi_v4_bpe_64k": { + "lang": "fi", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 151, + "unique_chunks": 1088693, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 309337, + "eval_chars_per_token": 4.2973423806398845, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 26429, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.00015218152211958423, + "marker_variant_extra_slots": 193, + "whitespace_only_vocab_entries": 23 + }, + "fi_v5_bpe_16k": { + "lang": "fi", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 104, + "unique_chunks": 1096254, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 359338, + "eval_chars_per_token": 3.6993777446304037, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4043, + "space_dup_pairs": 3, + "space_dup_vocab_frac": 0.0003387725142566766, + "marker_variant_extra_slots": 100, + "whitespace_only_vocab_entries": 13 + }, + "fi_v5_bpe_32k": { + "lang": "fi", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 118, + "unique_chunks": 1096254, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 321599, + "eval_chars_per_token": 4.133492330510978, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 10615, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 139, + "whitespace_only_vocab_entries": 17 + }, + "fi_v5_bpe_64k": { + "lang": "fi", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 146, + "unique_chunks": 1096254, + "eval_docs": 500, + "eval_chars": 1329327, + "eval_tokens": 292729, + "eval_chars_per_token": 4.541152396926851, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 26128, + "space_dup_pairs": 5, + "space_dup_vocab_frac": 0.00015218152211958423, + "marker_variant_extra_slots": 186, + "whitespace_only_vocab_entries": 23 + }, + "ru_plain_bpe_16k": { + "lang": "ru", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17710, + "train_seconds": 73, + "unique_chunks": 914115, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1581242, + "eval_chars_per_token": 3.4742563124430035, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 13727, + "space_dup_pairs": 1594, + "space_dup_vocab_frac": 0.18001129305477132, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 19 + }, + "ru_plain_bpe_32k": { + "lang": "ru", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33710, + "train_seconds": 87, + "unique_chunks": 914115, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1434863, + "eval_chars_per_token": 3.8286860836191328, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27624, + "space_dup_pairs": 3199, + "space_dup_vocab_frac": 0.18979531296351232, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 26 + }, + "ru_plain_bpe_64k": { + "lang": "ru", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 115, + "unique_chunks": 914115, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1323451, + "eval_chars_per_token": 4.150996145682765, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 54727, + "space_dup_pairs": 7016, + "space_dup_vocab_frac": 0.21354436158879928, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 35 + }, + "ru_plain_mingram_64k": { + "lang": "ru", + "pretokenizer": "plain", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65701, + "train_seconds": 323, + "unique_chunks": 914115, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1309273, + "eval_chars_per_token": 4.195946910995644, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 54839, + "space_dup_pairs": 6772, + "space_dup_vocab_frac": 0.20614602517465488, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 34 + }, + "ru_v4_bpe_16k": { + "lang": "ru", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 72, + "unique_chunks": 798887, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1623063, + "eval_chars_per_token": 3.384736143945121, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4785, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00045169668567556886, + "marker_variant_extra_slots": 124, + "whitespace_only_vocab_entries": 19 + }, + "ru_v4_bpe_32k": { + "lang": "ru", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 86, + "unique_chunks": 798887, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1466242, + "eval_chars_per_token": 3.746748490358345, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 12456, + "space_dup_pairs": 7, + "space_dup_vocab_frac": 0.00041529471092521727, + "marker_variant_extra_slots": 162, + "whitespace_only_vocab_entries": 26 + }, + "ru_v4_bpe_64k": { + "lang": "ru", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 118, + "unique_chunks": 798887, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1349757, + "eval_chars_per_token": 4.0700955801673935, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 30158, + "space_dup_pairs": 9, + "space_dup_vocab_frac": 0.0002739267398152516, + "marker_variant_extra_slots": 240, + "whitespace_only_vocab_entries": 35 + }, + "ru_v4_mingram_64k": { + "lang": "ru", + "pretokenizer": "v4", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 287, + "unique_chunks": 798887, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1338386, + "eval_chars_per_token": 4.104675332826255, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 33235, + "space_dup_pairs": 7, + "space_dup_vocab_frac": 0.0002130573733069548, + "marker_variant_extra_slots": 223, + "whitespace_only_vocab_entries": 34 + }, + "ru_v5_bpe_16k": { + "lang": "ru", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 72, + "unique_chunks": 810580, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1558448, + "eval_chars_per_token": 3.525071096372802, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4629, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00045169668567556886, + "marker_variant_extra_slots": 120, + "whitespace_only_vocab_entries": 19 + }, + "ru_v5_bpe_32k": { + "lang": "ru", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 87, + "unique_chunks": 810580, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1398346, + "eval_chars_per_token": 3.928670014431335, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 12211, + "space_dup_pairs": 7, + "space_dup_vocab_frac": 0.00041529471092521727, + "marker_variant_extra_slots": 157, + "whitespace_only_vocab_entries": 26 + }, + "ru_v5_bpe_64k": { + "lang": "ru", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 109, + "unique_chunks": 810580, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1280060, + "eval_chars_per_token": 4.291705076324547, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 29651, + "space_dup_pairs": 9, + "space_dup_vocab_frac": 0.0002739267398152516, + "marker_variant_extra_slots": 234, + "whitespace_only_vocab_entries": 35 + }, + "ru_v5_mingram_64k": { + "lang": "ru", + "pretokenizer": "v5", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65709, + "train_seconds": 281, + "unique_chunks": 810580, + "eval_docs": 500, + "eval_chars": 5493640, + "eval_tokens": 1268581, + "eval_chars_per_token": 4.330539397957245, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 32819, + "space_dup_pairs": 7, + "space_dup_vocab_frac": 0.0002130606157451795, + "marker_variant_extra_slots": 210, + "whitespace_only_vocab_entries": 33 + }, + "ar_plain_bpe_16k": { + "lang": "ar", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17710, + "train_seconds": 54, + "unique_chunks": 751910, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 200745, + "eval_chars_per_token": 3.516271887220105, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 13326, + "space_dup_pairs": 1578, + "space_dup_vocab_frac": 0.1782044042913608, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 19 + }, + "ar_plain_bpe_32k": { + "lang": "ar", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33710, + "train_seconds": 67, + "unique_chunks": 751910, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 184869, + "eval_chars_per_token": 3.818238861031325, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 26535, + "space_dup_pairs": 3108, + "space_dup_vocab_frac": 0.1843963215663008, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 33 + }, + "ar_plain_bpe_64k": { + "lang": "ar", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 94, + "unique_chunks": 751910, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 173915, + "eval_chars_per_token": 4.058729839289308, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 52492, + "space_dup_pairs": 6489, + "space_dup_vocab_frac": 0.1975041850555471, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 51 + }, + "ar_v4_bpe_16k": { + "lang": "ar", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 53, + "unique_chunks": 680572, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 206873, + "eval_chars_per_token": 3.412112745500863, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 6708, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00045169668567556886, + "marker_variant_extra_slots": 153, + "whitespace_only_vocab_entries": 18 + }, + "ar_v4_bpe_32k": { + "lang": "ar", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 68, + "unique_chunks": 680572, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 189574, + "eval_chars_per_token": 3.7234747380969964, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 15736, + "space_dup_pairs": 8, + "space_dup_vocab_frac": 0.0004746225267716769, + "marker_variant_extra_slots": 195, + "whitespace_only_vocab_entries": 31 + }, + "ar_v4_bpe_64k": { + "lang": "ar", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 96, + "unique_chunks": 680572, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 177700, + "eval_chars_per_token": 3.9722791221159257, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 35193, + "space_dup_pairs": 10, + "space_dup_vocab_frac": 0.00030436304423916847, + "marker_variant_extra_slots": 292, + "whitespace_only_vocab_entries": 49 + }, + "ar_v5_bpe_16k": { + "lang": "ar", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 60, + "unique_chunks": 694494, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 199239, + "eval_chars_per_token": 3.542850546328781, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 6557, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00045169668567556886, + "marker_variant_extra_slots": 150, + "whitespace_only_vocab_entries": 18 + }, + "ar_v5_bpe_32k": { + "lang": "ar", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 76, + "unique_chunks": 694494, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 181667, + "eval_chars_per_token": 3.885537824701239, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 15498, + "space_dup_pairs": 8, + "space_dup_vocab_frac": 0.0004746225267716769, + "marker_variant_extra_slots": 194, + "whitespace_only_vocab_entries": 31 + }, + "ar_v5_bpe_64k": { + "lang": "ar", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 99, + "unique_chunks": 694494, + "eval_docs": 500, + "eval_chars": 705874, + "eval_tokens": 169625, + "eval_chars_per_token": 4.161379513633014, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 34491, + "space_dup_pairs": 10, + "space_dup_vocab_frac": 0.00030436304423916847, + "marker_variant_extra_slots": 290, + "whitespace_only_vocab_entries": 46 + }, + "ko_plain_bpe_16k": { + "lang": "ko", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17710, + "train_seconds": 122, + "unique_chunks": 2312248, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 265333, + "eval_chars_per_token": 1.9627298526756942, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 12664, + "space_dup_pairs": 2605, + "space_dup_vocab_frac": 0.2941840767927724, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 19 + }, + "ko_plain_bpe_32k": { + "lang": "ko", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33710, + "train_seconds": 129, + "unique_chunks": 2312248, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 245102, + "eval_chars_per_token": 2.12473582426908, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 25484, + "space_dup_pairs": 5078, + "space_dup_vocab_frac": 0.30127558587956094, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 27 + }, + "ko_plain_bpe_64k": { + "lang": "ko", + "pretokenizer": "plain", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 167, + "unique_chunks": 2312248, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 228040, + "eval_chars_per_token": 2.2837089984213295, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 51209, + "space_dup_pairs": 10201, + "space_dup_vocab_frac": 0.3104854664434637, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 33 + }, + "ko_plain_mingram_64k": { + "lang": "ko", + "pretokenizer": "plain", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65710, + "train_seconds": 405, + "unique_chunks": 2312248, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 226708, + "eval_chars_per_token": 2.2971267004252165, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 51292, + "space_dup_pairs": 10089, + "space_dup_vocab_frac": 0.30707654847055244, + "marker_variant_extra_slots": 0, + "whitespace_only_vocab_entries": 29 + }, + "ko_v4_bpe_16k": { + "lang": "ko", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 101, + "unique_chunks": 2105977, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 278089, + "eval_chars_per_token": 1.8726990280090188, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4462, + "space_dup_pairs": 3, + "space_dup_vocab_frac": 0.0003387725142566766, + "marker_variant_extra_slots": 160, + "whitespace_only_vocab_entries": 18 + }, + "ko_v4_bpe_32k": { + "lang": "ko", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 120, + "unique_chunks": 2105977, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 254713, + "eval_chars_per_token": 2.044563881702151, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 11390, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 238, + "whitespace_only_vocab_entries": 25 + }, + "ko_v4_bpe_64k": { + "lang": "ko", + "pretokenizer": "v4", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 154, + "unique_chunks": 2105977, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 235666, + "eval_chars_per_token": 2.2098096458547265, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27423, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.00018261782654350109, + "marker_variant_extra_slots": 328, + "whitespace_only_vocab_entries": 32 + }, + "ko_v4_mingram_64k": { + "lang": "ko", + "pretokenizer": "v4", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 363, + "unique_chunks": 2105977, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 234635, + "eval_chars_per_token": 2.2195196795022056, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 28770, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.00018261782654350109, + "marker_variant_extra_slots": 289, + "whitespace_only_vocab_entries": 28 + }, + "ko_v5_bpe_16k": { + "lang": "ko", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 16000, + "vocab_size": 17711, + "train_seconds": 101, + "unique_chunks": 2116893, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 268773, + "eval_chars_per_token": 1.9376090604338978, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 4364, + "space_dup_pairs": 3, + "space_dup_vocab_frac": 0.0003387725142566766, + "marker_variant_extra_slots": 154, + "whitespace_only_vocab_entries": 18 + }, + "ko_v5_bpe_32k": { + "lang": "ko", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 32000, + "vocab_size": 33711, + "train_seconds": 116, + "unique_chunks": 2116893, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 244894, + "eval_chars_per_token": 2.1265404624041424, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 11305, + "space_dup_pairs": 4, + "space_dup_vocab_frac": 0.00023731126338583845, + "marker_variant_extra_slots": 229, + "whitespace_only_vocab_entries": 24 + }, + "ko_v5_bpe_64k": { + "lang": "ko", + "pretokenizer": "v5", + "method": "bpe", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 149, + "unique_chunks": 2116893, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 225761, + "eval_chars_per_token": 2.3067624611868305, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 27267, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.00018261782654350109, + "marker_variant_extra_slots": 324, + "whitespace_only_vocab_entries": 32 + }, + "ko_v5_mingram_64k": { + "lang": "ko", + "pretokenizer": "v5", + "method": "mingram", + "additional_vocab_size": 64000, + "vocab_size": 65711, + "train_seconds": 365, + "unique_chunks": 2116893, + "eval_docs": 500, + "eval_chars": 520777, + "eval_tokens": 224740, + "eval_chars_per_token": 2.3172421464803774, + "roundtrip_failures": 0, + "distinct_alpha_words_with_own_token": 28536, + "space_dup_pairs": 6, + "space_dup_vocab_frac": 0.00018261782654350109, + "marker_variant_extra_slots": 280, + "whitespace_only_vocab_entries": 28 + } +} \ No newline at end of file diff --git a/marker_experiments/paper.md b/marker_experiments/paper.md new file mode 100644 index 00000000..1a9c6dda --- /dev/null +++ b/marker_experiments/paper.md @@ -0,0 +1,387 @@ +# Boundary markers: one canonical word form, and better compression, for SCRIPT tokenizers + +**Status:** draft. The FineWiki 1 GB BPE results (§4.1) are complete; the MinGram +half (§4.2) is running and its table is marked pending. Results at smaller scale +(§4.3) come from an earlier per-script variant of the scheme and are labelled so. + +## Abstract + +Subword vocabularies built over space-separated writing systems spend a large +fraction of their capacity representing the same word twice — once with a leading +space and once without (`' the'` and `'the'`). Across six languages at 1 GB each we +measure this at **18–30% of a 32,768-entry vocabulary**, and at 16k vocabulary on web +text it accounts for **64% of all emitted tokens**. We replace the leading-space +convention with an explicit boundary marker `<|>`: spans are delimited, and the single +space between two adjacent delimited spans is elided at encode time and reconstructed +at decode time from the resulting pair of touching markers. *Which* units get delimited +decides everything. Delimiting words alone costs **−13.75%** compression on average; +adding punctuation brings it to **−2.99%**; adding digits makes it **+2.14%**, beating +the baseline in **all six languages at 1 GB** (range +0.88% to +3.77%) while reducing duplicate +vocabulary pairs from thousands to fewer than ten, with zero roundtrip failures and no +increase in training cost. The result is a tokenizer with one canonical form per word, +~20% of vocabulary reclaimed, and *better* compression than the convention it replaces. + +## 1. The duplication + +SCRIPT encoding (`ScriptEncodingV3`, registry name `scriptenc3_cb`) pretokenizes into +script/category runs and lets a lone space attach to the following span, producing the +`' word'` / `'word'` pair. Measured on FineWiki, 1 GB per language, 32,768 additional +vocabulary, BPE: + +| lang | duplicate pairs | share of vocabulary | +|---|---|---| +| en | 3,196 | 18.5% | +| de | 3,226 | 18.7% | +| fi | 3,835 | 22.2% | +| ru | 3,297 | 19.1% | +| ar | 3,159 | 18.3% | +| ko | 5,244 | **30.4%** | + +At 16k vocabulary on 80M characters of FineWeb the same measurement gives 1,792 pairs +occupying 22.4% of the vocabulary and accounting for **64.23%** of all emitted tokens — +the largest being `' .'`/`'.'` (700,931), `' ,'`/`','` (680,504), `' the'`/`'the'` +(639,584). + +A caveat we had to accept early: this duplication is not *waste* in compression terms. +Both forms are used and both earn their slots; §3 shows the baseline beating an early +marker scheme by 7.5% despite spending 30% of its vocabulary on duplicates. The +motivation is representational — one form per word, capacity freed for distinct content. +That the final scheme also compresses *better* is a separate result, and it took three +iterations to reach. + +## 2. Method + +One atomic token `<|>` is added. Text is grouped into maximal script/category runs, then +collected into units: + +- **word** — a maximal run of characters from *any* space-using script + (`DEFAULT_SCRIPTS_LM_WITH_SPACES`, category LM), **merged across script changes**; +- **punct** — `combines_with_spaces` but not letters (V3's `(✭, PSF)` blocks); +- **digit** — category `N`; +- **space** — exactly one space character; +- **other** — everything else (multi-space runs, newlines, Han, emoji). + +Marker placement: + +- **word** spans are delimited on **both** sides, unconditionally — the point of the + scheme: a word looks the same regardless of what precedes it; +- **punct** and **digit** units are delimited only on a side whose adjacent single space + was elided; +- **other** units and non-elided spaces are emitted exactly as the baseline emits them. + +A single space is elided when **both** neighbours are delimited. Decoding is then: + +``` +<|> immediately followed by <|> -> emit exactly one space +a lone <|> -> emit nothing (structural boundary) +``` + +### 2.1 Why spans merge across scripts + +Merging word runs across script changes is what makes the invariant hold without +exceptions. If each script run were delimited separately, `latin` immediately followed by +Cyrillic `кириллица` would put two unconditional markers face to face — indistinguishable +from an elided space — and decode would fabricate one. We found this empirically: Greek +letters used as identifiers (`sπ`, `upperΔ`, `Δx`) broke 5 of 500 held-out code documents. +Merging first means two delimited word spans can never be adjacent, so a touching pair is +unambiguously an elided space: + +``` +latinкириллица -> <|>latin | кириллица<|> (one span, no inner marker) +123 latinкириллица 123 -> 123<|> | <|>latin | кириллица<|> | <|>123 +``` + +Inside a span the baseline's script split is preserved — the marker rides the first and +last chunk — so no BPE merge crosses a script change the baseline would forbid. + +### 2.2 Why punctuation and digits are asymmetric + +Delimiting punctuation unconditionally would make `a,b` encode as +`<|>a<|> <|>,<|> <|>b<|>`: touching markers at both junctions, indistinguishable from +`a , b`. Marking only on a side that actually had a space keeps the invariant: + +``` +a,b -> <|>a<|> , <|>b<|> a, b -> <|>a<|> ,<|> <|>b<|> +a ,b -> <|>a<|> <|>, <|>b<|> a = b -> <|>a<|> <|>=<|> <|>b<|> +``` + +The cost is up to four variants per mark (`,` `,<|>` `<|>,` `<|>,<|>`). For punctuation +this is cheap because the set is genuinely closed: 146–225 slots at 32,768 vocabulary, +under 0.7%. + +**For digits it is not, and this is a real cost we initially mis-stated.** A digit *unit* +is a whole run, so the marked set is one entry per distinct *number*, not per digit. At +32,768 vocabulary the `bnd_wpd` runs of §4.1 spend, on pure-digit entries: + +| lang | plain entries | `bnd_wpd` entries | distinct numbers covered | slots lost to variants | +|---|---|---|---|---| +| en | 1,426 | 1,682 | 589 (vs 1,426) | **1,093 (3.17%)** | +| de | 1,128 | 1,416 | 522 (vs 1,128) | 894 (2.59%) | +| ru | 786 | 1,269 | 445 (vs 786) | 824 (2.39%) | +| ko | 927 | 1,082 | 441 (vs 927) | 641 (1.86%) | + +That is the very duplication the scheme exists to remove, reintroduced for numbers: more +entries spent, less than half the coverage. `digit_handling` bounds the markable set by +splitting digit runs so only the run's first and last *group* can carry a marker — 10 +markable strings under `SPLIT`, 1110 under `RTL3` (`pretokenizer.py` registers exactly +those). §4.4 measures what that is worth: the bound holds exactly, and it buys very +little. + +### 2.3 Merge constraint and chunking + +`bpe_merge_allowed` forbids any merge across two touching markers. Without it BPE learns +tokens like `'<|>the<|><|>'` that swallow the dangling half of the next span's opening +marker, reintroducing per-word duplication keyed on what *follows*. + +Units are not fused across an elided space. Given the merge constraint, fusing admits +exactly the same legal merges while inflating an early prototype's corpus from 335k to +1.76M unique chunks and BPE training from 40s to 442s — ~10× cost for no benefit, since +decoding reads the flat atomic stream and markers still touch across a chunk boundary. + +## 3. Design iterations + +| variant | prose 16k chars/token | vs plain | why | +|---|---|---|---| +| plain `scriptenc3_cb` | 3.9909 | — | baseline | +| rename the merged space token | 3.9909 | 0.00% | no-op: BPE's *first* learned merge already reattaches the space to the same token id | +| hard chunk boundary, space kept | 2.4201 | −39.4% | not the proposal: forbidding merges across a retained space destroys compression | +| words only | 3.6920 | −7.49% | right mechanism, wrong scope | +| + punctuation | 3.9266 | −1.61% | removes bare space tokens after punctuation | +| + digits | see §4 | positive | removes the last non-elided spaces | + +The words-only → punctuation step closed 79% of the gap *without improving word coverage +at all* (7,717 vs 7,752 single-token words). The gain came entirely from eliminating bare +space tokens: with only words delimited, `', b'` emits an unmarked `,`, a standalone +`' '`, and `<|>b<|>`, where the baseline absorbs the space into `' b'`. + +The punctuation → digits step was found by measuring which spaces the scheme *failed* to +elide: + +| domain | single spaces | elided | not elided | of those, digit-adjacent | +|---|---|---|---|---| +| code | 42,242 | 87.5% | 12.5% | **97.9%** | +| prose | 287,220 | 96.7% | 3.3% | **98.7%** | + +Digits are ~98% of every miss. The cause is structural: `script_category_v3` folds +`L/M → LM`, `Z/Cc → ZC`, `So → So` and `P/S/Cf → PSF` but leaves category `N` alone, so +digits are neither letters nor `combines_with_spaces` and were invisible to the scheme. +Delimiting them drops non-elided spaces to 0.3% (code) and 0.0% (prose). + +## 4. Results + +`ScriptEncodingV3` with `enforce_char_boundaries=True` throughout; the four pretokenizers +differ *only* in which units carry a boundary, so the comparison isolates the scheme. +Metric is characters per token on held-out documents (higher is better); roundtrip is +verified on every held-out document of every cell. + +### 4.1 FineWiki, 1 GB per language, 32,768 vocabulary, BPE + +Languages are those of `FINEWIKI_HYBRID6_CORPORA`. `normalize_whitespace` is applied as +the registry's finewiki loader does, so multi-space runs are absent by construction. +Held-out slice is the last 500 documents. + +| lang | script | plain | `bnd_w` | `bnd_wp` | `bnd_wpd` | +|---|---|---|---|---|---| +| en | Latin | 3.8310 | −15.97% | −2.90% | **+3.77%** | +| de | Latin | 4.1110 | −12.56% | −3.59% | **+1.60%** | +| fi | Latin | 3.9836 | −12.64% | −2.94% | **+2.02%** | +| ru | Cyrillic | 3.7955 | −13.72% | −2.35% | **+2.67%** | +| ar | Arabic | 3.9838 | −12.41% | −2.67% | **+1.91%** | +| ko | Hangul | 2.2310 | −15.21% | −3.48% | **+0.88%** | +| **mean** | | | **−13.75%** | **−2.99%** | **+2.14%** | + +**Zero roundtrip failures across all 24 cells.** The progression is tight across scripts: +`bnd_w` clusters in −12 to −16%, `bnd_wp` in −2.4 to −3.6%, `bnd_wpd` positive everywhere. + +Vocabulary structure at the same setting: + +| lang | plain dup pairs | plain vocab on dups | `bnd_wpd` pairs | `bnd_wpd` variant slots | +|---|---|---|---|---| +| en | 3,196 | 18.5% | 4 | 213 | +| de | 3,226 | 18.7% | 4 | 188 | +| fi | 3,835 | 22.2% | 4 | 146 | +| ru | 3,297 | 19.1% | 5 | 185 | +| ar | 3,159 | 18.3% | 6 | 198 | +| ko | 5,244 | 30.4% | 4 | 218 | + +Training cost is not a penalty: `bnd_wpd` is at or below baseline time in every language +(en 160s vs 152s, de 286s vs 329s, fi 322s vs 353s, ru 231s vs 241s, ar 200s vs 206s, ko +388s vs 456s) and yields fewer unique chunks (ko 8.60M vs 9.47M). + +One metric misleads if read directly: *distinct words with their own token* falls from +~27k (plain) to ~11–16k (`bnd_wpd`). That is not lost coverage — the baseline +double-counts, holding `' the'` and `'the'` as two entries for one word, while the marker +vocabulary holds one canonical form. + +### 4.2 MinGram, and the effect of scale + +At 1 GB a MinGram cell takes ~26 min, which this environment's ~30–60 min working-tree +wipes reliably destroyed: one cell of 24 survived (`en` plain, 3.8738, +1.12% over BPE). +The comparison was therefore rerun at 250M characters with both trainers, three languages +spanning the script families, and evaluation documents withheld from training. Rows here +are internally comparable and **not** comparable to §4.1. + +| lang | trainer | plain | `bnd_w` | `bnd_wp` | `bnd_wpd` | +|---|---|---|---|---|---| +| en | BPE | 3.7580 | −15.24% | −3.42% | **+3.38%** | +| en | MinGram | 3.8019 | −15.64% | −3.78% | **+3.02%** | +| ru | BPE | 3.8094 | −11.66% | −1.35% | **+2.39%** | +| ru | MinGram | 3.8636 | −12.02% | −1.63% | **+2.27%** | +| ko | BPE | 2.1793 | −17.04% | −4.23% | **−0.67%** | +| ko | MinGram | 2.2055 | −17.79% | −5.05% | **−1.51%** | + +MinGram gains +1.17% (en), +1.42% (ru) and +1.20% (ko) over BPE on the baseline, and +preserves the `bnd_w` < `bnd_wp` < `bnd_wpd` ordering in every language. The effect is +therefore not a BPE artifact. MinGram consistently helps the baseline slightly more than +it helps `bnd_wpd`, shrinking the margin by 0.1–0.8pp. + +**Korean changes sign with scale: +0.88% at 1 GB, −0.67% at 250M.** This is the clearest +statement of the mechanism in §5. Hangul gives Korean the shortest spans of the six +languages (2.18 chars/token here), so the two marker tokens are proportionally heavy +overhead, and more data is needed before the marker vocabulary covers enough words to +amortise them. The claim in §4.1 that `bnd_wpd` beats the baseline in all six languages +is a statement about 1 GB, not a general one. + +### 4.4 Digit handling + +en, 250M characters, 32,768 vocabulary, BPE, evaluation documents withheld from training. +Both sides use the same `digit_handling`, so each row is a matched comparison; rows are +**not** comparable to each other, since splitting digits changes absolute compression for +everyone. + +| `digit_handling` | plain | `bnd_wpd` | gap | digit-variant slots | distinct numbers | +|---|---|---|---|---|---| +| `None` | 3.7580 | 3.8849 | **+3.38%** | 1,083 | 608 | +| `SPLIT` | 3.4437 | 3.5714 | **+3.71%** | **34** | 14 | +| `RTL3` | 3.7096 | 3.8345 | **+3.37%** | 1,478 | 1,114 | + +The bound behaves exactly as predicted. `SPLIT` collapses the variant waste from 1,083 +slots to 34 — at most ten digits times four forms — and `RTL3` lands at 1,114 distinct +numbers against a ceiling of 1110. + +**Recovering ~1,050 vocabulary slots is worth only +0.33pp.** We expected removing a 3.17% +vocabulary tax to move the headline; it does not. This is the same lesson as §1: the +duplicated number entries are wasteful *and* still used, and low-frequency number tokens +carry little mass. Vocabulary waste and compression loss are not interchangeable. + +The practically useful result is the invariance: **the boundary advantage sits between ++3.37% and +3.71% regardless of digit policy.** Practitioners who split digits for +arithmetic reasons lose nothing, and those who do not lose nothing either. `RTL3` costs +the baseline 1.3% absolute against `SPLIT`'s 8.4%, so it is the better default — and that +choice is independent of this work. + +### 4.3 Smaller scale, earlier per-script variant + +These predate the span merging of §2.1 and delimited each script run separately. They are +included because they establish scale and domain behaviour, and because the mixed +prose+code corpus is the only place code was studied. + +**FineWeb English prose**, 80M chars, 1000 held-out docs, `+punct` variant: + +| vocab | plain BPE | +punct | plain MinGram | +punct | +|---|---|---|---|---| +| 16k | 3.9909 | −1.61% | 4.0474 | −1.99% | +| 32k | 4.2611 | −1.03% | 4.3012 | −1.23% | +| 64k | 4.4412 | −0.86% | 4.4678 | −1.06% | + +**Mixed prose + code**, 40M FineWeb + 40M code (codeparrot Python files, rosetta-code +snippets), no whitespace normalization, 500 held-out docs per domain, `+digits` against +baseline: + +| trainer | vocab | mixed | prose | code | +|---|---|---|---|---| +| BPE | 16k | −0.10% | +0.05% | −0.56% | +| BPE | 32k | +0.67% | +0.79% | +0.33% | +| BPE | 64k | **+1.17%** | +1.13% | +1.29% | +| MinGram | 64k | +1.07% | +0.98% | +1.33% | + +The span-merged design at 1 GB (+3.77% for en at 32k) outperforms the per-script design at +100M (+2.63% for en at 32k), so removing the inner boundary cost nothing and gained. + +## 5. Analysis + +**Overshoot does not substitute for the right boundary set.** Sweeping MinGram's BPE-init +overshoot at 16k on the words-only variant: + +| f | plain | words-only | gap | words-only single-token words | +|---|---|---|---|---| +| 1.10 | 4.0455 | 3.7286 | −7.83% | 8,674 | +| 1.15 | 4.0474 | 3.7282 | −7.89% | 8,856 | +| 1.25 | 4.0470 | 3.7252 | −7.95% | 8,964 | + +A 2.5× larger candidate pool bought 290 words and made compression marginally *worse*. +MinGram does recover a one-time ~900 words by pruning dead BPE intermediates, but that is +~900 of a ~5,000-word deficit. + +**The deficit was bare space tokens, not vocabulary accounting.** Bucketing the token +stream on held-out code, the words+punct variant's +3,256-token deficit versus baseline +decomposes as whitespace +1,708 (52%), alpha +721, punct +434, marker-only +382, digit ++11. The whitespace term is *not* indentation — pure space tokens of length > 1 are 1.24% +of code tokens with identical counts under both schemes, and BPE folds them into +`"\n "`-style tokens running at 4.59 chars/token. It is the digit case: in `1 item` the +baseline absorbs the space into `' item'` while an undelimited digit blocks elision. +Counting cases where the following unit is delimited but the preceding is not gives 1,834, +against the measured +1,708. + +**Korean is the informative weak case.** It has the largest duplicate tax to reclaim +(30.4%) and the smallest gain from reclaiming it (+0.88%). Hangul syllable blocks give +Korean by far the lowest absolute compression (2.23 vs 3.80–4.11 chars/token), so word +spans are short and the two marker tokens are proportionally heavier. The rule that falls +out: the scheme pays in proportion to span length relative to its two markers. + +## 6. Limitations + +- **No language-modelling evaluation.** Everything here is compression and vocabulary + structure. Whether a canonical word form helps or hurts downstream quality is the + obvious next experiment and is not addressed. +- **Space-using scripts only.** All six languages use spaces; the scheme does nothing for + Han, Thai or other spaceless scripts, which keep baseline behaviour. +- **Open-set scripts are undelimited**, so ~2% of single spaces remain non-elided in mixed + text. CJK-heavy corpora were not studied. +- **Gains shrink where spans are short** relative to the marker pair, and can go negative: + Korean is +0.88% at 1 GB but −0.67% at 250M (§4.2). The scheme needs enough data to + amortise two marker tokens per span, and short-span scripts need more of it. +- **Marker-only tokens are pure overhead**: 382 emissions, 0.27% of code tokens, carrying + no characters. +- **§4.3 uses first-N sampling**, not the registry's seeded reservoir sample over the full + source, and its corpora are 80M chars against `fineweb_en_5gb`'s 5×10⁹. +- **One roundtrip failure per code cell in §4.3, baseline included**: `U+F8FF` is absent + from the V3 `char_encoding` and is dropped. Pre-existing and unrelated, but zero failures + is unreachable on that corpus without a script-config fix. +- **Single vocabulary size at 1 GB.** §4.1 is 32,768 only; the smaller-scale runs show the + advantage growing with vocabulary, but that is not verified at 1 GB. +- **§4.1 trains on its own evaluation slice.** The held-out documents are the last 500 of + the stream, but the corpus was built from the whole stream, so they are ~1.3% of the + training data. The leak is identical for every pretokenizer, so the *gaps* in §4.1 are + unaffected; absolute chars/token is optimistic for all four alike. Fixed for the digit + axis (§4.4), which withholds them. +- **§4.1 ran with `digit_handling=None`**, so it pays the digit-variant tax of §2.2. §4.4 + measures the effect of removing it on English: +0.33pp, so the §4.1 figures are close to + what a digit-split configuration would give. §4.4 is English-only, one vocabulary size, + BPE only. + +## 7. Reproduction + +``` +marker_experiments/ + boundary_pretokenizer.py # BoundaryScriptPretokenizer, boundary_targets config + test_boundary.py # 412 tests + finewiki1gb_grid.py # 4.1/4.2 grid: resumable, commits each cell with its tokenizer + finewiki1gb_result.json # 4.1/4.2 numbers + multilang_grid.py # earlier per-script 100M multilingual grid + multilang_result.json + prior_results.json # 4.3 and 5 numbers + tokenizers/ # every trained tokenizer +``` + +The three variants are one class differing only in `boundary_targets`, and produce distinct +`hash()` values so they cannot collide in the pretokenized-corpus cache — a trap the +earlier prototypes fell into, since `Pretokenizer.hash()` is config-derived and ignores +behaviour. + +A note on running this environment: the container clears the working tree every ~30–60 +minutes and caps disk, so the grid streams text rather than staging it, frees each +language's corpora when done, retries transient CDN failures, and commits and pushes every +finished cell. Reading only shard `000_00000` silently under-reads languages whose first +shard is smaller than the budget (Arabic 483M, Korean ~734M); the runner lists and reads +all shards. diff --git a/marker_experiments/paper/README.md b/marker_experiments/paper/README.md new file mode 100644 index 00000000..8d178efe --- /dev/null +++ b/marker_experiments/paper/README.md @@ -0,0 +1,98 @@ +# ACL-format source + +`acl_latex.tex` + `custom.bib`. Content mirrors `../paper.md`, which stays the working draft. + +## Building + +The ACL style files are **not** included and could not be fetched from this environment +(GitHub raw returned 404, the API was unreachable). Download them into this directory: + +``` +https://github.com/acl-org/acl-style-files + -> acl.sty + -> acl_natbib.bst +``` + +Then: + +``` +pdflatex acl_latex +bibtex acl_latex +pdflatex acl_latex +pdflatex acl_latex +``` + +**This has never been compiled.** There is no TeX toolchain in the container, so the source +is unverified: expect to fix at least the usual first-build complaints (missing style file, +unicode in the Cyrillic/Greek examples, table widths in the two-column layout). The Cyrillic +and Greek examples in §3.1 are written with ASCII placeholders (``, `\Delta`, `\pi`) +rather than literal glyphs precisely because `pdflatex` will not typeset them without extra +packages; switch to `xelatex` with a Unicode font if you want the real characters. + +Remove `[review]` from `\usepackage[review]{acl}` for a camera-ready build. + +## Style and citation alignment + +Conventions are taken from this repository's own paper-table generators +(`paper_utils/hybrid/`, `paper_utils/unigram/`): `booktabs` rules with `\cmidrule` group +separators, a `\relchange` macro for relative-change columns, languages spelled out +(English, German, Finnish, Russian, Arabic, Korean) rather than ISO codes, and method names +matching the MinGram tables (BPE, MinGram). + +The three papers from this line of work are cited, with metadata fetched and verified from +arXiv during preparation: + +| Paper | Authors | arXiv | +|---|---|---| +| BPE Stays on SCRIPT | Land & Arnett | 2505.24689 | +| Which Pieces Does Unigram Tokenization Really Need? | Land & Pinter | 2512.12641 | +| MinGram | Land | 2606.27019 | + +Two corrections were made against the first draft of this bibliography: + +- **MinGram was misattributed to Schmidt et al.** MinGram is Land (2026); Schmidt et al. + (2024) is PathPiece, whose minimum-increase pruning rule MinGram's optional prune + criterion follows. Both are now cited, for the right things. +- **SCRIPT was not cited at all**, despite the baseline throughout being its + `scriptenc3_cb` pretokenizer. It is now cited in the introduction and §2. + +## Bibliography + +`custom.bib` is the supplied curated bibliography, used verbatim with its ACL Anthology +keys. Four entries were appended at the end because the paper cites them and they were not +in the supplied file: + +| Key | Status | +|---|---| +| `land2026mingram` | title/author/date verified against arXiv (2606.27019) | +| `penedo2024fineweb` | **unverified** — FineWiki is supplied, FineWeb is not | +| `codeparrot` | **unverified** | +| `rosettacode` | **unverified** | + +Three claims lost their citation because the supplied bibliography has no entry for them, +and were rewritten rather than left hanging on an invented reference: + +- Gage's original BPE — the claim now rests on `sennrich-etal-2016-neural` alone. +- GPT-2 byte-level BPE — replaced by `tokencontributions-gpt4`, which covers + pre-tokenization and punctuation in production tokenizers and is a better fit anyway. +- Llama digit splitting — the sentence now states the arithmetic rationale without + attributing it to a specific model. + +59 bib entries, 34 cited, no undefined keys. + +## Content differences from `paper.md` + +- Adds §1 Introduction, which the Markdown draft lacks, framing the contribution as the + choice of *which* units to delimit rather than the marker mechanism itself. +- Adds citations throughout; the Markdown draft has none. +- Adds a Related Work section absent from the Markdown draft. SuperBPE + (`liu2025superbpe`) and Boundless BPE (`schmidt2025boundless`) are the nearest prior art + and were missing entirely: both extend tokens *across* the pre-tokenization boundary, + where this work keeps pre-tokens word-sized and makes the boundary explicit instead. The + section also positions the duplication measurement against scaffold-token pruning + (PickyBPE, Scaffold-BPE, Magikarp) and states plainly that compression is an incomplete + proxy, citing both sides of that debate. +- Tables are `booktabs`; the six-language and MinGram tables are unchanged numerically. +- The mixed prose+code results are compressed to a paragraph (§5.4) since they come from the + earlier per-script variant. +- Author block is `Anonymous`; acknowledgments are a placeholder. diff --git a/marker_experiments/paper/acl_latex.tex b/marker_experiments/paper/acl_latex.tex new file mode 100644 index 00000000..b4f3b207 --- /dev/null +++ b/marker_experiments/paper/acl_latex.tex @@ -0,0 +1,508 @@ +\documentclass[11pt]{article} + +% Requires the ACL style files (acl.sty, acl_natbib.bst) from +% https://github.com/acl-org/acl-style-files -- see README.md in this directory. +\usepackage[review]{acl} + +\usepackage{times} +\usepackage{latexsym} +\usepackage{booktabs} +\usepackage{amsmath} +\usepackage{graphicx} +\usepackage{inconsolata} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{microtype} +\usepackage{xcolor} + +\newcommand{\bnd}[1]{\texttt{bnd\_#1}} +\newcommand{\mk}{\texttt{<|>}} +% Relative change against the baseline, following the convention of the MinGram tables. +\newcommand{\relchange}[1]{{\footnotesize #1}} + +\title{Boundary Markers: One Canonical Word Form, and Better Compression,\\ + for Script-Aware Subword Tokenization} + +\author{Anonymous} + +\begin{document} +\maketitle + +\begin{abstract} +Subword vocabularies built over space-separated writing systems spend a large fraction of +their capacity representing the same word twice --- once with a leading space and once +without (\texttt{' the'} and \texttt{'the'}). Across six languages at 1\,GB each we measure +this at 18--30\% of a 32{,}768-entry vocabulary, and at 16k vocabulary on web text it +accounts for 64\% of all emitted tokens. We replace the leading-space convention with an +explicit boundary marker \mk{}: spans are delimited, and the single space between two +adjacent delimited spans is elided at encode time and reconstructed at decode time from the +resulting pair of touching markers. \emph{Which} units get delimited decides the outcome. +Delimiting words alone costs $-13.75\%$ compression on average; adding punctuation brings it +to $-2.99\%$; adding digits reaches $+2.14\%$, beating the baseline in all six languages at +1\,GB (range $+0.88\%$ to $+3.77\%$), while reducing duplicate vocabulary pairs from +thousands to fewer than ten, with zero roundtrip failures and no increase in training cost. +The result is a tokenizer with one canonical form per word, roughly 20\% of vocabulary +reclaimed, and better compression than the convention it replaces. Extending the same +argument to case with two caps codes removes a further 29.1\% of the vocabulary at no +measurable compression cost. +\end{abstract} + +\section{Introduction} +\label{sec:intro} + +Byte-pair encoding \citep{sennrich-etal-2016-neural} and unigram tokenizers +\citep{kudo-2018-subword,land2025piecesdoesunigramtokenization} are conventionally applied to text in which a +word's leading space is absorbed into the word's token. SentencePiece \citep{kudo-richardson-2018-sentencepiece} makes this +explicit with a substituted space character, and byte-level tokenizers inherit it; the +interaction of pre-tokenization with punctuation in production tokenizers is surveyed by +\citet{tokencontributions-gpt4}. The consequence is that most content words occupy two vocabulary entries: one +with the leading space, one without. + +We ask what happens if the convention is replaced by an explicit boundary token. Our +contribution is not the marker itself but the finding that the choice of \emph{which} units +to delimit spans a range from $-13.75\%$ to $+2.14\%$ mean compression relative to the +baseline --- the mechanism is identical throughout, and only the scope changes. + +We also report two negative results that constrain how the primary result should be read. +First, the duplication we remove is not waste in compression terms: the baseline outperforms +an early version of our scheme by 7.5\% while spending 30\% of its vocabulary on duplicate +pairs. Second, reclaiming vocabulary does not straightforwardly convert into compression: +removing a 3.17\% digit-variant tax buys only $+0.33$pp (\S\ref{sec:digits}). + +\section{Related work} +\label{sec:related} + +Two recent lines of work also target the pre-tokenization boundary. SuperBPE +\citep{liu2025superbpe} lets merges run \emph{across} whitespace, so single tokens may span +several words; Boundless BPE \citep{schmidt2025boundless} likewise removes the +pre-tokenization barrier and merges across pre-token boundaries. Both increase the span a +token may cover. Our direction is the opposite: we keep pre-tokens word-sized and instead +make the boundary itself explicit, so a word has one form rather than two. The two are not +in conflict --- an explicit boundary token is what a superword vocabulary would need in order +to remain invertible --- but they trade different things, and we do not evaluate the +combination. + +A second thread removes vocabulary entries that exist only as merge scaffolding. PickyBPE +\citep{chizhov2024bpegetspickyefficient} and Scaffold-BPE \citep{scaffold_bpe} both prune +intermediate ``junk'' tokens during training, and \citet{land-bartolo-2024-fishing} show +such entries survive into deployed models as under-trained tokens. The duplication we +measure is a different phenomenon --- both members of a \texttt{' the'}/\texttt{'the'} pair +are heavily used (\S\ref{sec:duplication}) --- but the two share a motivation, and +\S\ref{sec:digits} shows that reclaiming unused capacity does not by itself buy compression. + +On evaluation, \citet{galle-2019-investigating} argues that fewer tokens on a fixed budget +predicts downstream quality, while \citet{schmidt-etal-2024-tokenization}, +\citet{zouhar-etal-2023-tokenization} and its counterexamples +\citep{cognetta-etal-2024-two}, and \citet{lotz-etal-2025-beyond} all find compression an +incomplete proxy. We report compression only, and \S\ref{sec:limitations} treats this as the +principal limitation. + +\section{The duplication} +\label{sec:duplication} + +We build on SCRIPT encoding \citep{scriptbpe}, which segments text into runs of shared +Unicode script and category and encodes each character as a block/index token pair. Its +pretokenizer lets a lone space attach to the following run, producing the +\texttt{' word'}/\texttt{'word'} pair. Table~\ref{tab:dup} measures the resulting +duplication on FineWiki \citep{penedo2025finewiki}, 1\,GB per language, 32{,}768 additional +vocabulary, BPE. + +\begin{table}[t] +\centering +\small +\begin{tabular}{lrr} +\toprule +Language & Duplicate pairs & \% of vocabulary \\ +\midrule +English & 3{,}196 & 18.5 \\ +German & 3{,}226 & 18.7 \\ +Finnish & 3{,}835 & 22.2 \\ +Russian & 3{,}297 & 19.1 \\ +Arabic & 3{,}159 & 18.3 \\ +Korean & 5{,}244 & \textbf{30.4} \\ +\bottomrule +\end{tabular} +\caption{Vocabulary spent on \texttt{' X'}/\texttt{'X'} duplicate pairs.} +\label{tab:dup} +\end{table} + +At 16k vocabulary on 80M characters of FineWeb \citep{penedo2024fineweb} the same +measurement gives 1{,}792 pairs occupying 22.4\% of the vocabulary and accounting for +\textbf{64.23\%} of all emitted tokens, the largest being \texttt{' .'}/\texttt{'.'} +(700{,}931), \texttt{' ,'}/\texttt{','} (680{,}504) and \texttt{' the'}/\texttt{'the'} +(639{,}584). + +We stress a caveat established early in this work: the duplication is not \emph{waste} in +compression terms. Both forms are used and both earn their slots. The motivation is +representational --- one form per word, and capacity freed for distinct content. That the +final scheme also compresses better is a separate result, reached only after three +iterations (\S\ref{sec:iterations}). + +\section{Method} +\label{sec:method} + +One atomic token \mk{} is added to the pretokenizer. Text is grouped into maximal +script/category runs, which are collected into units: \textbf{word} (a maximal run from any +space-using script, category LM, merged across script changes), \textbf{punct}, +\textbf{digit} (Unicode category N), \textbf{space} (exactly one space character), and +\textbf{other} (multi-space runs, newlines, Han, emoji). + +Word spans are delimited on both sides unconditionally --- this is the point of the scheme, +since it gives a word one form regardless of context. Punctuation and digit units are +delimited only on a side whose adjacent single space was elided. Other units and non-elided +spaces are emitted exactly as the baseline emits them. A single space is elided when both +neighbours are delimited, so decoding is: +\begin{quote}\small +\mk{} immediately followed by \mk{} $\rightarrow$ emit one space \\ +a lone \mk{} $\rightarrow$ emit nothing +\end{quote} + +\subsection{Spans merge across scripts} +\label{sec:spans} + +Merging word runs across script changes is what makes the invariant hold without exception. +If each script run were delimited separately, Latin immediately followed by Cyrillic would +place two unconditional markers face to face --- indistinguishable from an elided space --- +and decoding would fabricate one. We found this empirically: Greek letters used as +identifiers (\texttt{s$\pi$}, \texttt{upper$\Delta$}, \texttt{$\Delta$x}) broke 5 of 500 +held-out code documents. Merging first means two delimited word spans can never be adjacent: +\begin{quote}\small +\texttt{latin} $\rightarrow$ \texttt{\mk{}latin | \mk{}} +\end{quote} +Inside a span the baseline's script split is preserved, the marker riding the first and last +chunk, so no merge crosses a script change the baseline would forbid. + +\subsection{Punctuation and digits are asymmetric} +\label{sec:asymmetric} + +Delimiting punctuation unconditionally would encode \texttt{a,b} as +\texttt{\mk{}a\mk{} \mk{},\mk{} \mk{}b\mk{}}: touching markers at both junctions, and so +indistinguishable from \texttt{a , b}. Marking only on a side that had a space preserves the +invariant, at the cost of up to four variants per mark (\texttt{,}\ \texttt{,\mk{}}\ +\texttt{\mk{},}\ \texttt{\mk{},\mk{}}). For punctuation this is cheap because the set is +genuinely closed: 146--225 slots at 32{,}768 vocabulary, under 0.7\%. + +For digits it is not, and \S\ref{sec:digits} quantifies it: a digit \emph{unit} is a whole +run, so the marked set has one entry per distinct \emph{number}, not per digit. + +\subsection{Merge constraint and chunking} + +Merges across two touching markers are forbidden. Without this, BPE learns tokens such as +\texttt{'\mk{}the\mk{}\mk{}'} that swallow the dangling half of the next span's opening +marker, reintroducing per-word duplication keyed on what \emph{follows}. Units are not fused +across an elided space: given the constraint, fusing admits exactly the same legal merges +while inflating an early prototype's corpus from 335k to 1.76M unique chunks and BPE +training from 40s to 442s. + +\subsection{Scope: when one marker glyph suffices} +\label{sec:onemarker} + +We use a single glyph for both sides of a span. This is sound here only because +pretokenization fixes which chunk each marker belongs to \emph{before} any search runs: +\texttt{\mk{}word\mk{}} and \texttt{\mk{}.} are separate chunks, and both BPE merging and +the MinGram dynamic program operate strictly within a chunk. No segmentation is ever +offered a choice about which piece covers a given marker. + +That condition is doing real work, and it is worth stating because it does not hold for +every tokenizer. Where the segmenter tiles a flat glyph stream --- a minimum-piece method +with no pre-tokenization barrier, as has been reported for at least one production +tokenizer \citep{tokencontributions-claude} --- a single glyph is ambiguous: a +word-\emph{initial} piece form \texttt{\mk{}x} can absorb a marker that \emph{closed} the +preceding word, because nothing in the glyph distinguishes the two roles. The segmenter +takes whichever tiling is cheaper, which need not be the intended one. Direction must then +be recovered either from a distinct pair of glyphs, opening and closing, or from a +directional constraint on matching. + +Two glyphs cost one extra atomic token and, since order already distinguishes +\texttt{\mk{},} from \texttt{,\mk{}}, no additional piece-form variants. They do not, +however, remove either of the other two constraints in this section: a learned token can +still end in a dangling opening marker, so the seam restriction is still required, and an +opening marker meeting a closing one is still indistinguishable from an elided space, so +span merging (\S\ref{sec:spans}) is still required. Choosing one glyph over two is +therefore a consequence of pretokenizing into chunks, not an independent simplification. + +\section{Design iterations} +\label{sec:iterations} + +Table~\ref{tab:iterations} traces the path to the final scheme; each row was measured before +the next was designed. + +\begin{table}[t] +\centering +\small +\begin{tabular}{lrr} +\toprule +Variant & chars/tok & vs.\ base \\ +\midrule +baseline & 3.9909 & --- \\ +rename merged space token & 3.9909 & $0.00\%$ \\ +hard boundary, space kept & 2.4201 & $-39.4\%$ \\ +words only & 3.6920 & $-7.49\%$ \\ +\ \ + punctuation & 3.9266 & $-1.61\%$ \\ +\ \ + digits & \S\ref{sec:results} & positive \\ +\bottomrule +\end{tabular} +\caption{English prose, 16k vocabulary, BPE. Renaming the merged space token is a no-op +because BPE's \emph{first} learned merge already reattaches the space to the same token id.} +\label{tab:iterations} +\end{table} + +The words-only $\rightarrow$ punctuation step closed 79\% of the gap \emph{without improving +word coverage at all} (7{,}717 vs.\ 7{,}752 single-token words). The gain came entirely from +eliminating bare space tokens: with only words delimited, \texttt{', b'} emits an unmarked +comma, a standalone space, and \texttt{\mk{}b\mk{}}, where the baseline absorbs the space +into \texttt{' b'}. + +The punctuation $\rightarrow$ digits step was found by measuring which spaces the scheme +failed to elide: 12.5\% on code and 3.3\% on prose, of which 97.9\% and 98.7\% respectively +were adjacent to a digit. Delimiting digits drops non-elided spaces to 0.3\% and 0.0\%. + +\section{Results} +\label{sec:results} + +All pretokenizers share the same script encoding and character-boundary policy and differ +\emph{only} in which units carry a boundary, so comparisons isolate the scheme. The metric is +characters per token on held-out documents; roundtrip is verified on every held-out document +of every cell. + +\subsection{Six languages at 1\,GB} +\label{sec:multiling} + +Languages follow the six-language set used throughout this line of work +\citep{land2025piecesdoesunigramtokenization,land2026mingram} --- English, German, Finnish, Russian, Arabic and +Korean --- spanning Latin, Cyrillic, Arabic and Hangul, and drawn from the monolingual +corpora of \citet{chang2026goldfishmonolinguallanguagemodels}. +Whitespace normalization is applied as the corpus loader does, so multi-space runs are absent +by construction. Table~\ref{tab:main} reports the main result. + +\begin{table}[t] +\centering +\small +\begin{tabular}{llrrr} +\toprule +& \multicolumn{1}{c}{Baseline} & \multicolumn{3}{c}{Relative change (\%)} \\ +\cmidrule(lr){2-2} \cmidrule(lr){3-5} +Language & plain & \bnd{w} & \bnd{wp} & \bnd{wpd} \\ +\midrule +English & 3.8310 & \relchange{$-15.97$} & \relchange{$-2.90$} & $\mathbf{+3.77}$ \\ +German & 4.1110 & \relchange{$-12.56$} & \relchange{$-3.59$} & $\mathbf{+1.60}$ \\ +Finnish & 3.9836 & \relchange{$-12.64$} & \relchange{$-2.94$} & $\mathbf{+2.02}$ \\ +Russian & 3.7955 & \relchange{$-13.72$} & \relchange{$-2.35$} & $\mathbf{+2.67}$ \\ +Arabic & 3.9838 & \relchange{$-12.41$} & \relchange{$-2.67$} & $\mathbf{+1.91}$ \\ +Korean & 2.2310 & \relchange{$-15.21$} & \relchange{$-3.48$} & $\mathbf{+0.88}$ \\ +\midrule +Mean & & \relchange{$-13.75$} & \relchange{$-2.99$} & $\mathbf{+2.14}$ \\ +\bottomrule +\end{tabular} +\caption{FineWiki, 1\,GB per language, 32{,}768 vocabulary, BPE. Baseline in chars/token; +variants as \% relative to it. Zero roundtrip failures across all 24 cells.} +\label{tab:main} +\end{table} + +Duplicate pairs fall from 3{,}159--5{,}244 to 4--10, at a marker-variant cost of 146--324 +slots. Training cost is not a penalty: \bnd{wpd} is at or below baseline time in every +language, and produces fewer unique chunks. + +One metric misleads if read directly. \emph{Distinct words with their own token} falls from +roughly 27k (plain) to 11--16k (\bnd{wpd}). This is not lost coverage: the baseline +double-counts, holding \texttt{' the'} and \texttt{'the'} as two entries for one word, while +the marker vocabulary holds one canonical form. + +\subsection{MinGram, and the effect of scale} +\label{sec:mingram} + +To check that the effect is not specific to greedy merge training, we repeat the comparison +with MinGram \citep{land2026mingram}, a minimum-token-count unigram trainer initialised from +BPE, whose optional pruning criterion follows the minimum-increase rule of PathPiece +\citep{schmidt-etal-2024-tokenization}. We use 250M characters, where cells complete reliably, with +evaluation documents withheld from training (Table~\ref{tab:mingram}). + +\begin{table}[t] +\centering +\small +\begin{tabular}{llrrrr} +\toprule +& & \multicolumn{1}{c}{Baseline} & \multicolumn{3}{c}{Relative change (\%)} \\ +\cmidrule(lr){3-3} \cmidrule(lr){4-6} +Language & Trainer & plain & \bnd{w} & \bnd{wp} & \bnd{wpd} \\ +\midrule +English & BPE & 3.7580 & \relchange{$-15.24$} & \relchange{$-3.42$} & $+3.38$ \\ +English & MinGram & 3.8019 & \relchange{$-15.64$} & \relchange{$-3.78$} & $+3.02$ \\ +Russian & BPE & 3.8094 & \relchange{$-11.66$} & \relchange{$-1.35$} & $+2.39$ \\ +Russian & MinGram & 3.8636 & \relchange{$-12.02$} & \relchange{$-1.63$} & $+2.27$ \\ +Korean & BPE & 2.1793 & \relchange{$-17.04$} & \relchange{$-4.23$} & $-0.67$ \\ +Korean & MinGram & 2.2055 & \relchange{$-17.79$} & \relchange{$-5.05$} & $-1.51$ \\ +\bottomrule +\end{tabular} +\caption{250M characters, 32{,}768 vocabulary. Not comparable to Table~\ref{tab:main}.} +\label{tab:mingram} +\end{table} + +The two trainers also differ in inference-time segmentation --- greedy merge replay versus a +minimum-token dynamic program --- a distinction shown to matter independently of the +vocabulary \citep{uzan-etal-2024-greed}. MinGram gains $+1.17\%$ (en), $+1.42\%$ (ru) and +$+1.20\%$ (ko) over BPE on the baseline and +preserves the \bnd{w} $<$ \bnd{wp} $<$ \bnd{wpd} ordering everywhere, so the effect is not a +BPE artifact. It helps the baseline slightly more than \bnd{wpd}, costing 0.1--0.8pp of +margin. + +\paragraph{Korean changes sign with scale.} Korean is $+0.88\%$ at 1\,GB but $-0.67\%$ at +250M. Hangul gives Korean the shortest spans of the six languages (2.18 chars/token here), so +two marker tokens per span are proportionally heavy and need more data to amortise. The claim +that \bnd{wpd} beats the baseline in all six languages is a statement about 1\,GB, not a +general one. Korean's segmentation is known to reward sub-syllabic treatment +\citep{lee-etal-2025-jamo}, and per-language differences in tokenized length are the subject +of a substantial fairness literature +\citep{petrov2023language,ahia-etal-2023-languages,arnett-etal-2024-bit,velayuthan-sarveswaran-2025-egalitarian}; +we do not attempt to correct for byte premium here. + +\subsection{Digit handling} +\label{sec:digits} + +A digit unit is a whole run, so every distinct number acquires up to four marked forms. At +32{,}768 vocabulary this costs English 1{,}093 slots (3.17\%): \bnd{wpd} spends 1{,}682 +pure-digit entries against the baseline's 1{,}426 while covering 589 distinct numbers against +1{,}426. That is the duplication the scheme exists to remove, reintroduced for numbers. + +Splitting digit runs bounds the markable set, since only a run's first and last group can +carry a marker: 10 markable strings when split to single digits, 1110 under right-to-left +grouping in threes. Digit splitting is normally adopted for arithmetic rather than compression reasons. Because splitting changes absolute compression +for everyone, each row of Table~\ref{tab:digits} is a matched comparison and rows are not +comparable to each other. + +\begin{table}[t] +\centering +\small +\begin{tabular}{lrrrr} +\toprule +Digits & plain & \bnd{wpd} & gap & var.\ slots \\ +\midrule +none & 3.7580 & 3.8849 & $+3.38\%$ & 1{,}083 \\ +split & 3.4437 & 3.5714 & $+3.71\%$ & \textbf{34} \\ +rtl3 & 3.7096 & 3.8345 & $+3.37\%$ & 1{,}478 \\ +\bottomrule +\end{tabular} +\caption{English, 250M characters, 32{,}768 vocabulary, BPE, evaluation withheld.} +\label{tab:digits} +\end{table} + +The bound behaves as predicted: splitting collapses variant waste from 1{,}083 slots to 34 +--- ten digits times four forms --- and right-to-left grouping lands at 1{,}114 distinct +numbers against a ceiling of 1110. \textbf{But recovering roughly 1{,}050 vocabulary slots is +worth only $+0.33$pp.} This is the same lesson as \S\ref{sec:duplication}: the duplicated +entries are wasteful \emph{and} still used, and low-frequency number tokens carry little +mass. Vocabulary waste and compression loss are not interchangeable. + +The practically useful result is invariance: the boundary advantage sits between $+3.37\%$ +and $+3.71\%$ regardless of digit policy, so the scheme composes with whatever digit handling +was already chosen. + +\subsection{Caps codes} +\label{sec:caps} + +The same argument applies to case: without intervention a vocabulary holds \texttt{The} and +\texttt{the}, \texttt{NASA} and \texttt{nasa} as separate entries. Following the older +Claude tokenizer \citep{tokencontributions-claude}, we add two codes --- shift for +title case, caps-lock for all caps --- and emit the lowercased span after them, so +\texttt{The} is exactly \texttt{the} plus one code. Whole spans only; mixed case such as +\texttt{GaN} or \texttt{WiFi} stays literal. + +Unicode case mapping is not a bijection, so invertibility is verified per span rather than +assumed: U+0130 lowercases to two characters, U+1E9E uppercases to \texttt{SS}, and +titlecase digraphs such as U+01C5 do not survive a lower/upper round trip. Each candidate is +re-transformed and compared, and anything that does not reproduce the source exactly stays +literal. Checked over all 2{,}842 cased letters in Unicode, and over 2{,}000 FineWiki +documents in English, German, Russian, Greek and Turkish (4.7M characters), with no +round-trip failures. + +\begin{table}[t] +\centering +\small +\begin{tabular}{lrrr} +\toprule +& chars/tok & case pairs & alpha entries \\ +\midrule +\bnd{wpd} & 3.8849 & 5{,}017 & 26{,}661 \\ +\ \ + caps codes & 3.8832 & \textbf{91} & 21{,}819 \\ +\bottomrule +\end{tabular} +\caption{English, 250M characters, 32{,}768 vocabulary, BPE, evaluation withheld. Case +pairs counts entries with a distinct-cased counterpart also present.} +\label{tab:caps} +\end{table} + +Caps codes remove case duplication almost entirely --- 5{,}017 pairs to 91, or 29.1\% of the +vocabulary to 0.5\% --- at a compression cost of $-0.04\%$, which is noise. This is the +third instance of the same pattern: reclaiming vocabulary does not convert into compression, +but here it does not cost anything either, so the canonical-case form is effectively free. +Note that this is the reverse of the digit case, where bounding the markable set was +\emph{necessary} to avoid a penalty; case has only two productive forms per word, so the +duplication is bounded to begin with. + +\subsection{Mixed prose and code} +\label{sec:code} + +An earlier per-script variant of the scheme was evaluated on a mixed corpus of 40M characters +of web prose and 40M characters of code \citep{codeparrot,rosettacode}, without whitespace +normalization. At 64k vocabulary the digit-complete variant reached $+1.17\%$ overall, +$+1.13\%$ on prose and $+1.29\%$ on code. Indentation, which might be expected to dominate +code, does not: multi-space runs are 1.24\% of emitted code tokens with identical counts +under both schemes. + +\section{Analysis} +\label{sec:analysis} + +\paragraph{Larger candidate pools do not substitute for the right boundary set.} Sweeping the +BPE-initialisation overshoot \citep{land2026mingram} over $1.10$, $1.15$ and $1.25$ on the +words-only variant moves the gap from $-7.83\%$ to $-7.95\%$ --- a 2.5$\times$ larger +candidate pool buys 290 words and makes compression marginally worse. + +\paragraph{The deficit was bare space tokens.} Bucketing the token stream on held-out code, +the words-plus-punctuation variant's $+3{,}256$-token deficit decomposes as whitespace +$+1{,}708$ (52\%), alphabetic $+721$, punctuation $+434$, marker-only $+382$, digit $+11$. +The whitespace term is not indentation but the digit case: in \texttt{1 item} the baseline +absorbs the space into \texttt{' item'} while an undelimited digit blocks elision. Counting +cases where the following unit is delimited but the preceding is not gives 1{,}834, against +the measured $+1{,}708$. + +\paragraph{Cost scales with span length.} Korean has the largest duplicate tax to reclaim +(30.4\%) and the smallest gain from reclaiming it. The scheme pays two marker tokens per +span, so its benefit is governed by span length relative to that fixed cost. + +\section{Limitations} +\label{sec:limitations} + +No language-modelling evaluation is reported. Everything here is compression and vocabulary +structure; whether a canonical word form helps or hurts downstream quality is untested, and +prior work repeatedly cautions that the two need not agree +\citep{rust-etal-2021-good,zouhar-etal-2023-tokenization,cognetta-etal-2024-two,ali-etal-2024-tokenizer,lotz-etal-2025-beyond}. +Morphological alignment, another axis on which such a change could help or hurt +\citep{bostrom-durrett-2020-byte,arnett2025evaluatingmorphologicalalignmenttokenizers}, is +likewise not measured. + +All six languages use spaces; the scheme does nothing for Han, Thai or other spaceless +scripts, which retain baseline behaviour, and morphologically complex languages may respond +differently \citep{arnett-bergen-2025-language,vemula-etal-2025-rethinking}. Open-set scripts are undelimited, so roughly 2\% of +single spaces remain non-elided in mixed text, and CJK-heavy corpora were not studied. + +Gains shrink where spans are short relative to the marker pair, and can go negative: Korean is +$+0.88\%$ at 1\,GB but $-0.67\%$ at 250M (\S\ref{sec:mingram}). + +The single-glyph design assumes a pretokenizer that pins markers to chunks +(\S\ref{sec:onemarker}). We have not measured the scheme under a segmenter that tiles a +flat stream, where a second glyph or a directional matching constraint would be required; +whether the compression results carry over to that setting is untested. + +Table~\ref{tab:main} uses a single vocabulary size, and its corpus was built over the same +documents used for evaluation (roughly 1.3\% of training data). The leak is identical for +every pretokenizer, so the reported gaps are unaffected while absolute chars/token is +optimistic for all four alike; later experiments withhold the evaluation slice. The digit +axis is English-only, one vocabulary size, BPE only. + +\section*{Acknowledgments} + +Placeholder. + +\bibliography{custom} + +\end{document} diff --git a/marker_experiments/paper/custom.bib b/marker_experiments/paper/custom.bib new file mode 100644 index 00000000..29ae22e0 --- /dev/null +++ b/marker_experiments/paper/custom.bib @@ -0,0 +1,896 @@ +@inproceedings{schmidt-etal-2024-tokenization, + title = "Tokenization Is More Than Compression", + author = "Schmidt, Craig W. and + Reddy, Varshini and + Zhang, Haoran and + Alameddine, Alec and + Uzan, Omri and + Pinter, Yuval and + Tanner, Chris", + editor = "Al-Onaizan, Yaser and + Bansal, Mohit and + Chen, Yun-Nung", + booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing", + month = nov, + year = "2024", + address = "Miami, Florida, USA", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.emnlp-main.40/", + doi = "10.18653/v1/2024.emnlp-main.40", + pages = "678--702" +} + +@inproceedings{uzan-etal-2024-greed, + title = "Greed is All You Need: An Evaluation of Tokenizer Inference Methods", + author = "Uzan, Omri and Schmidt, Craig W. and Tanner, Chris and Pinter, Yuval", + booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)", + month = aug, + year = "2024", + address = "Bangkok, Thailand", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.acl-short.73/", + doi = "10.18653/v1/2024.acl-short.73", + pages = "813--822" +} + +@inproceedings{zouhar-etal-2023-tokenization, + title = "Tokenization and the Noiseless Channel", + author = "Zouhar, Vil{\'e}m and + Meister, Clara and + Gastaldi, Juan and + Du, Li and + Sachan, Mrinmaya and + Cotterell, Ryan", + editor = "Rogers, Anna and + Boyd-Graber, Jordan and + Okazaki, Naoaki", + booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = jul, + year = "2023", + address = "Toronto, Canada", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.acl-long.284/", + doi = "10.18653/v1/2023.acl-long.284", + pages = "5184--5207" +} + +@inproceedings{cognetta-etal-2024-two, + title = "Two Counterexamples to Tokenization and the Noiseless Channel", + author = "Cognetta, Marco and + Zouhar, Vil{\'e}m and + Moon, Sangwhan and + Okazaki, Naoaki", + editor = "Calzolari, Nicoletta and + Kan, Min-Yen and + Hoste, Veronique and + Lenci, Alessandro and + Sakti, Sakriani and + Xue, Nianwen", + booktitle = "Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)", + month = may, + year = "2024", + address = "Torino, Italia", + publisher = "ELRA and ICCL", + url = "https://aclanthology.org/2024.lrec-main.1469/", + pages = "16897--16906" +} + +@article{nllb-24, + author = "{NLLB Team} and Costa-juss{\`a}, Marta R. and Cross, James and {\c{C}}elebi, Onur and Elbayad, Maha and Heafield, Kenneth and Heffernan, Kevin and Kalbassi, Elahe and Lam, Janice and Licht, Daniel and Maillard, Jean and Sun, Anna and Wang, Skyler and Wenzek, Guillaume and Youngblood, Al and Akula, Bapi and Barrault, Loic and Gonzalez, Gabriel Mejia and Hansanti, Prangthip and Hoffman, John and Jarrett, Semarley and Sadagopan, Kaushik and Rowe, Dirk and Spruit, Shannon and Tran, Chau and Andrews, Pierre and Ayan, Necip Fazil and Bhosale, Shruti and Edunov, Sergey and Fan, Angela and Gao, Cynthia and Goswami, Vedanuj and Guzm{\'a}n, Francisco and Koehn, Philipp and Mourachko, Alexandre and Ropers, Christophe and Saleem, Safiyyah and Schwenk, Holger and Wang, Jeff", + title = "Scaling neural machine translation to 200 languages", + journal = "Nature", + year = "2024", + volume = "630", + number = "8018", + pages = "841--846", + doi = "10.1038/s41586-024-07335-x", + url = "https://doi.org/10.1038/s41586-024-07335-x", +} + +% Summary: Foundational empirical comparison of BPE vs Unigram LM +% tokenization, on English and Japanese. Finds Unigram aligns better +% with morphology than BPE, with no major training-time speed difference, +% and that Unigram pretraining matches or beats BPE on downstream tasks. +% Verbatim from the paper: +% "the unigram LM method recovers subword units that align more closely +% with morphology and avoids problems stemming from BPE's greedy +% construction procedure" +% "the unigram LM method produces subword units that qualitatively align +% with morphology much better than those produced by BPE" +% "the unigram LM method recovers common affixes such as -ly, -s, pre-, +% and tri- while BPE does not, instead absorbing them into adjacent +% units (-cles) while also producing meaningless single-character units" +@inproceedings{bostrom-durrett-2020-byte, + title = "Byte Pair Encoding is Suboptimal for Language Model Pretraining", + author = "Bostrom, Kaj and Durrett, Greg", + booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2020", + month = nov, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2020.findings-emnlp.414/", + doi = "10.18653/v1/2020.findings-emnlp.414", + pages = "4617--4624", +} + +@inproceedings{yehezkel-pinter-2023-incorporating, + title = "Incorporating Context into Subword Vocabularies", + author = "Yehezkel, Shaked and + Pinter, Yuval", + editor = "Vlachos, Andreas and + Augenstein, Isabelle", + booktitle = "Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics", + month = may, + year = "2023", + address = "Dubrovnik, Croatia", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.eacl-main.45/", + doi = "10.18653/v1/2023.eacl-main.45", + pages = "623--635", +} + +% Summary: Tokenizer comparison for Telugu (agglutinative), Hindi, and +% English using small BERT models. Releases a Telugu gold morpheme +% segmentation dataset (600 derivational + 7000 inflectional forms). +% Headline finding is about downstream task performance, not directly +% alignment of segmentations: Unigram beats BPE on most downstream +% tasks for Telugu, and morphological alignment correlates moderately +% but is a weaker driver than algorithm choice. Useful as recent +% multilingual corroboration that the BPE/Unigram split persists for +% morphologically rich languages. +% Verbatim from the paper: +% "the choice of tokenizer algorithm is the most significant factor +% influencing performance, with Unigram-based tokenizers consistently +% outperforming BPE across most settings" +% "while better morphological alignment shows a moderate, positive +% correlation with performance on text classification and structure +% prediction tasks, its impact is secondary to the tokenizer algorithm" +@inproceedings{vemula-etal-2025-rethinking, + title = "Rethinking Tokenization for Rich Morphology: The Dominance of {U}nigram over {BPE} and Morphological Alignment", + author = "Vemula, Saketh Reddy and Dandapat, Sandipan and Sharma, Dipti and Krishnamurthy, Parameswari", + booktitle = "The 14th International Joint Conference on Natural Language Processing and The 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics", + month = dec, + year = "2025", + address = "Mumbai, India", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2025.ijcnlp-srw.20/", + doi = "10.18653/v1/2025.ijcnlp-srw.20", + pages = "232--252" +} + +@misc{asgari2025morphbpemorphoawaretokenizerbridging, + title={{MorphBPE}: A Morpho-Aware Tokenizer Bridging Linguistic Complexity for Efficient {LLM} Training Across Morphologies}, + author={Ehsaneddin Asgari and Yassine El Kheir and Mohammad Ali Sadraei Javaheri}, + year={2025}, + eprint={2502.00894}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2502.00894}, +} + +@inproceedings{land2025piecesdoesunigramtokenization, + title = "Which Pieces Does {U}nigram Tokenization Really Need?", + author = "Land, Sander and + Pinter, Yuval", + editor = "Liakata, Maria and + Moreira, Viviane P. and + Zhang, Jiajun and + Jurgens, David", + booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026", + month = jul, + year = "2026", + address = "San Diego, California, United States", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2026.findings-acl.316/", + doi = "10.18653/v1/2026.findings-acl.316", + pages = "6351--6360", + ISBN = "979-8-89176-395-1" +} + +@misc{penedo2025finewiki, + author = {Guilherme Penedo}, + title = {FineWiki}, + year = {2025}, + publisher = {Hugging Face Datasets}, + url = {https://huggingface.co/datasets/HuggingFaceFW/finewiki}, + urldate = {2025-10-20}, + note = {Source: Wikimedia Enterprise Snapshot API (https://api.enterprise.wikimedia.com/v2/snapshots). Text licensed under CC BY-SA 4.0 with attribution to Wikipedia contributors.} +} + +@misc{provilkov2020bpedropoutsimpleeffectivesubword, + title={BPE-Dropout: Simple and Effective Subword Regularization}, + author={Ivan Provilkov and Dmitrii Emelianenko and Elena Voita}, + year={2020}, + eprint={1910.13267}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/1910.13267}, +} + +@inproceedings{stephen2025morphtokeval, + title = "Evaluating Morphological Plausibility of Subword Tokenization via Statistical Alignment with Morpho-Syntactic Features", + author = "Stephen, Abishek and + Libovick{\'y}, Jind{\v{r}}ich", + editor = "Demberg, Vera and + Inui, Kentaro and + Marquez, Llu{\'i}s", + booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EACL} 2026", + month = mar, + year = "2026", + address = "Rabat, Morocco", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2026.findings-eacl.196/", + doi = "10.18653/v1/2026.findings-eacl.196", + pages = "3783--3791", + ISBN = "979-8-89176-386-9" +} + +@misc{arnett2025evaluatingmorphologicalalignmenttokenizers, + title={Evaluating Morphological Alignment of Tokenizers in 70 Languages}, + author={Catherine Arnett and Marisa Hudspeth and Brendan O'Connor}, + year={2025}, + eprint={2507.06378}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2507.06378}, +} + +@article{dempster1977maximum, + author = {A. P. Dempster and N. M. Laird and D. B. Rubin}, + journal = {Journal of the Royal Statistical Society. Series B (Methodological)}, + number = {1}, + pages = {1--38}, + publisher = {[Royal Statistical Society, Oxford University Press]}, + title = {Maximum Likelihood from Incomplete Data via the {EM} Algorithm}, + urldate = {2026-04-21}, + volume = {39}, + year = {1977} +} + +@inproceedings{kudo-richardson-2018-sentencepiece, + title = "{SentencePiece}: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing", + author = "Kudo, Taku and + Richardson, John", + editor = "Blanco, Eduardo and + Lu, Wei", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing: System Demonstrations", + month = nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/D18-2012/", + doi = "10.18653/v1/D18-2012", + pages = "66--71", +} + +@inproceedings{Kasai2001LinearTimeLC, + title={Linear-Time Longest-Common-Prefix Computation in Suffix Arrays and Its Applications}, + author={Toru Kasai and Gunho Lee and Hiroki Arimura and Setsuo Arikawa and Kunsoo Park}, + booktitle={Combinatorial Pattern Matching}, + pages={181--192}, + year={2001}, + organization={Springer} +} + +@misc{scriptbpe, + title={{BPE} Stays on {SCRIPT}: Structured Encoding for Robust Multilingual Pretokenization}, + author={Sander Land and Catherine Arnett}, + year={2025}, + eprint={2505.24689}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2505.24689}, +} + +@inproceedings{chizhov2024bpegetspickyefficient, + title = "{BPE} Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training", + author = "Chizhov, Pavel and + Arnett, Catherine and + Korotkova, Elizaveta and + Yamshchikov, Ivan P.", + editor = "Al-Onaizan, Yaser and + Bansal, Mohit and + Chen, Yun-Nung", + booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing", + month = nov, + year = "2024", + address = "Miami, Florida, USA", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.emnlp-main.925/", + doi = "10.18653/v1/2024.emnlp-main.925", + pages = "16587--16604" +} + +@inproceedings{arnett-etal-2024-bit, + title = {{A Bit of a Problem: Measurement Disparities in Dataset Sizes across Languages}}, + author = "Arnett, Catherine and + Chang, Tyler A. and + Bergen, Benjamin", + editor = "Melero, Maite and + Sakti, Sakriani and + Soria, Claudia", + booktitle = "Proceedings of the 3rd Annual Meeting of the Special Interest Group on Under-resourced Languages @ LREC-COLING 2024", + month = may, + year = "2024", + address = "Torino, Italia", + publisher = "ELRA and ICCL", + url = "https://aclanthology.org/2024.sigul-1.1/", + pages = "1--9", +} + +@inproceedings{sennrich-etal-2016-neural, + title = "Neural Machine Translation of Rare Words with Subword Units", + author = "Sennrich, Rico and + Haddow, Barry and + Birch, Alexandra", + editor = "Erk, Katrin and + Smith, Noah A.", + booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/P16-1162/", + doi = "10.18653/v1/P16-1162", + pages = "1715--1725" +} + +@inproceedings{velayuthan-sarveswaran-2025-egalitarian, + title = {{Egalitarian Language Representation in Language Models: It All Begins with Tokenizers}}, + author = "Velayuthan, Menan and + Sarveswaran, Kengatharaiyer", + editor = "Rambow, Owen and + Wanner, Leo and + Apidianaki, Marianna and + Al-Khalifa, Hend and + Eugenio, Barbara Di and + Schockaert, Steven", + booktitle = "Proceedings of the 31st International Conference on Computational Linguistics", + month = jan, + year = "2025", + address = "Abu Dhabi, UAE", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2025.coling-main.400/", + pages = "5987--5996", +} + +@misc{tokencontributions-gpt4, + title={Pre-tokenization on punctuation in {GPT-4}}, + author={Sander Land}, + year={2024}, + url = {https://tokencontributions.substack.com/p/pre-tokenization-on-punctuation-in}, + howpublished = "Blog Post" +} + +@misc{arnett-tokenizer-recycling, + title={{wHy DoNt YoU jUsT uSe ThE lLaMa ToKeNiZeR??}}, + author={Catherine Arnett}, + year={2024}, + url={https://huggingface.co/blog/catherinearnett/dangers-of-tokenizer-recycling}, + howpublished = "Blog Post" +} + +@inproceedings{oscar2020, + title = "A Monolingual Approach to Contextualized Word Embeddings for Mid-Resource Languages", + author = "Ortiz Suarez, Pedro Javier and + Romary, Laurent and + Sagot, Benoit", + booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", + month = jul, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/2020.acl-main.156", + pages = "1703--1714", +} + +@inproceedings{oscar2019, + author = {Pedro Javier {Ortiz Suarez} and Benoit Sagot and Laurent Romary}, + title = {Asynchronous pipelines for processing huge corpora on medium to low resource infrastructures}, + series = {Proceedings of the Workshop on Challenges in the Management of Large Corpora (CMLC-7) 2019. Cardiff, 22nd July 2019}, + editor = {Piotr Bański and Adrien Barbaresi and Hanno Biber and Evelyn Breiteneder and Simon Clematide and Marc Kupietz and Harald L{"u}ngen and Caroline Iliadi}, + publisher = {Leibniz-Institut f{"u}r Deutsche Sprache}, + address = {Mannheim}, + doi = {10.14618/ids-pub-9021}, + url = {http://nbn-resolving.de/urn:nbn:de:bsz:mh39-90215}, + pages = {9 -- 16}, + year = {2019}, + language = {en} +} + +@misc{nguyen2023culturaxcleanedenormousmultilingual, + title={{CulturaX}: A Cleaned, Enormous, and Multilingual Dataset for Large Language Models in 167 Languages}, + author={Thuat Nguyen and Chien Van Nguyen and Viet Dac Lai and Hieu Man and Nghia Trung Ngo and Franck Dernoncourt and Ryan A. Rossi and Thien Huu Nguyen}, + year={2023}, + eprint={2309.09400}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2309.09400}, +} + +@inproceedings{lee-etal-2025-jamo, + title = "Jamo-Level Subword Tokenization in Low-Resource {K}orean Machine Translation", + author = "Lee, Junyoung and + Cognetta, Marco and + Moon, Sangwhan and + Okazaki, Naoaki", + editor = "Ojha, Atul Kr. and + Liu, Chao-hong and + Vylomova, Ekaterina and + Pirinen, Flammie and + Washington, Jonathan and + Oco, Nathaniel and + Zhao, Xiaobing", + booktitle = "Proceedings of the Eighth Workshop on Technologies for Machine Translation of Low-Resource Languages (LoResMT 2025)", + month = may, + year = "2025", + address = "Albuquerque, New Mexico, U.S.A.", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2025.loresmt-1.8/", + pages = "66--80", + ISBN = "979-8-89176-230-5", +} + +@inproceedings{land-bartolo-2024-fishing, + title = "Fishing for {M}agikarp: Automatically Detecting Under-trained Tokens in Large Language Models", + author = "Land, Sander and + Bartolo, Max", + editor = "Al-Onaizan, Yaser and + Bansal, Mohit and + Chen, Yun-Nung", + booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing", + month = nov, + year = "2024", + address = "Miami, Florida, USA", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.emnlp-main.649/", + doi = "10.18653/v1/2024.emnlp-main.649", + pages = "11631--11646", +} + +@misc{wolf2020huggingfacestransformersstateoftheartnatural, + title={{HuggingFace's} Transformers: State-of-the-art Natural Language Processing}, + author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush}, + year={2020}, + eprint={1910.03771}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/1910.03771}, +} + +@misc{inan2017tying, + title={Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling}, + author={Hakan Inan and Khashayar Khosravi and Richard Socher}, + year={2017}, + eprint={1611.01462}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} + +@inproceedings{kudo-2018-subword, + title = "Subword Regularization: Improving Neural Network Translation Models with Multiple Subword Candidates", + author = "Kudo, Taku", + editor = "Gurevych, Iryna and + Miyao, Yusuke", + booktitle = "Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = jul, + year = "2018", + address = "Melbourne, Australia", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/P18-1007", + doi = "10.18653/v1/P18-1007", + pages = "66--75", +} + +@inproceedings{provilkov-etal-2020-bpe, + title = "{BPE}-Dropout: Simple and Effective Subword Regularization", + author = "Provilkov, Ivan and + Emelianenko, Dmitrii and + Voita, Elena", + editor = "Jurafsky, Dan and + Chai, Joyce and + Schluter, Natalie and + Tetreault, Joel", + booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", + month = jul, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2020.acl-main.170", + doi = "10.18653/v1/2020.acl-main.170", + pages = "1882--1892", +} + +@misc{mielke2021words, + title={Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in {NLP}}, + author={Sabrina J. Mielke and Zaid Alyafeai and Elizabeth Salesky and Colin Raffel and Manan Dey and Matthias Gallé and Arun Raja and Chenglei Si and Wilson Y. Lee and Benoît Sagot and Samson Tan}, + year={2021}, + eprint={2112.10508}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} + +@inproceedings{bis-etal-2021-much, + title = "Too Much in Common: Shifting of Embeddings in Transformer Language Models and its Implications", + author = "Bi{\'s}, Daniel and + Podkorytov, Maksim and + Liu, Xiuwen", + editor = "Toutanova, Kristina and + Rumshisky, Anna and + Zettlemoyer, Luke and + Hakkani-Tur, Dilek and + Beltagy, Iz and + Bethard, Steven and + Cotterell, Ryan and + Chakraborty, Tanmoy and + Zhou, Yichao", + booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies", + month = jun, + year = "2021", + url = "https://aclanthology.org/2021.naacl-main.403", + doi = "10.18653/v1/2021.naacl-main.403", + pages = "5117--5130", +} + +@inproceedings{liu2025superbpe, + title={{SuperBPE}: Space travel for language models}, + author={Alisa Liu and Jonathan Hayase and Valentin Hofmann and Sewoong Oh and Noah A Smith and Yejin Choi}, + booktitle={Second Conference on Language Modeling}, + year={2025}, + url={https://arxiv.org/abs/2503.13423} +} + +@misc{foroutan2025parityaware, + title={Parity-Aware Byte-Pair Encoding: Improving Cross-lingual Fairness in Tokenization}, + author={Negar Foroutan and Clara Meister and Debjit Paul and Joel Niklaus and Sina Ahmadi and Antoine Bosselut and Rico Sennrich}, + year={2025}, + eprint={2508.04796}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2508.04796}, +} + +@inproceedings{meister2026unigramlm, + title = {{UnigramLM}: An Attempt at Writing The Missing Manual}, + booktitle = {Proceedings of the Fourteenth International Conference on Learning Representations}, + author = {Meister, Clara}, + year = {2026}, + url = {https://cimeister.github.io/blog/unigramlm/}, + note = {ICLR 2026 Blogpost Track} +} + +@inproceedings{petrov2023language, + author = {Petrov, Aleksandar and La Malfa, Emanuele and Torr, Philip and Bibi, Adel}, + booktitle = {Advances in Neural Information Processing Systems}, + editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine}, + pages = {36963--36990}, + publisher = {Curran Associates, Inc.}, + title = {Language Model Tokenizers Introduce Unfairness Between Languages}, + url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/74bb24dca8334adce292883b4b651eda-Paper-Conference.pdf}, + volume = {36}, + year = {2023} +} + +@inproceedings{ahia-etal-2023-languages, + title = "Do All Languages Cost the Same? Tokenization in the Era of Commercial Language Models", + author = "Ahia, Orevaoghene and + Kumar, Sachin and + Gonen, Hila and + Kasai, Jungo and + Mortensen, David and + Smith, Noah and + Tsvetkov, Yulia", + editor = "Bouamor, Houda and + Pino, Juan and + Bali, Kalika", + booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing", + month = dec, + year = "2023", + address = "Singapore", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.emnlp-main.614/", + doi = "10.18653/v1/2023.emnlp-main.614", + pages = "9904--9923" +} + +@inproceedings{ali-etal-2024-tokenizer, + title = "Tokenizer Choice For {LLM} Training: Negligible or Crucial?", + author = "Ali, Mehdi and + Fromm, Michael and + Thellmann, Klaudia and + Rutmann, Richard and + L{\"u}bbering, Max and + Leveling, Johannes and + Klug, Katrin and + Ebert, Jan and + Doll, Niclas and + Buschhoff, Jasper and + Jain, Charvi and + Weber, Alexander and + Jurkschat, Lena and + Abdelwahab, Hammam and + John, Chelsea and + Ortiz Suarez, Pedro and + Ostendorff, Malte and + Weinbach, Samuel and + Sifa, Rafet and + Kesselheim, Stefan and + Flores-Herr, Nicolas", + editor = "Duh, Kevin and + Gomez, Helena and + Bethard, Steven", + booktitle = "Findings of the Association for Computational Linguistics: NAACL 2024", + month = jun, + year = "2024", + address = "Mexico City, Mexico", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.findings-naacl.247/", + doi = "10.18653/v1/2024.findings-naacl.247", + pages = "3907--3924", +} + +@inproceedings{lotz-etal-2025-beyond, + title = "Beyond Text Compression: Evaluating Tokenizers Across Scales", + author = "Lotz, Jonas F. and + Lopes, Ant{\'o}nio V. and + Peitz, Stephan and + Setiawan, Hendra and + Emili, Leonardo", + editor = "Che, Wanxiang and + Nabende, Joyce and + Shutova, Ekaterina and + Pilehvar, Mohammad Taher", + booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = jul, + year = "2025", + address = "Vienna, Austria", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2025.acl-long.1546/", + doi = "10.18653/v1/2025.acl-long.1546", + pages = "32155--32173", +} + +@inproceedings{arnett-bergen-2025-language, + title = "Why do language models perform worse for morphologically complex languages?", + author = "Arnett, Catherine and + Bergen, Benjamin", + editor = "Rambow, Owen and + Wanner, Leo and + Apidianaki, Marianna and + Al-Khalifa, Hend and + Eugenio, Barbara Di and + Schockaert, Steven", + booktitle = "Proceedings of the 31st International Conference on Computational Linguistics", + month = jan, + year = "2025", + address = "Abu Dhabi, UAE", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2025.coling-main.441/", + pages = "6607--6623" +} + +@inproceedings{rust-etal-2021-good, + title = "How Good is Your Tokenizer? On the Monolingual Performance of Multilingual Language Models", + author = "Rust, Phillip and + Pfeiffer, Jonas and + Vuli{\'c}, Ivan and + Ruder, Sebastian and + Gurevych, Iryna", + editor = "Zong, Chengqing and + Xia, Fei and + Li, Wenjie and + Navigli, Roberto", + booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)", + month = aug, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.acl-long.243/", + doi = "10.18653/v1/2021.acl-long.243", + pages = "3118--3135" +} + +@inproceedings{hofmann-etal-2021-superbizarre, + title = "Superbizarre Is Not Superb: Derivational Morphology Improves {BERT}{'}s Interpretation of Complex Words", + author = {Hofmann, Valentin and + Pierrehumbert, Janet and + Sch{\"u}tze, Hinrich}, + editor = "Zong, Chengqing and + Xia, Fei and + Li, Wenjie and + Navigli, Roberto", + booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)", + month = aug, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.acl-long.279/", + doi = "10.18653/v1/2021.acl-long.279", + pages = "3594--3608" +} + +@misc{tempus2026tokenisationconvexrelaxations, + title={Tokenisation via Convex Relaxations}, + author={Jan Tempus and Philip Whittington and Craig W. Schmidt and Dennis Komm and Tiago Pimentel}, + year={2026}, + eprint={2605.22821}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2605.22821}, +} + +@misc{chang2026goldfishmonolinguallanguagemodels, + title={Goldfish: Monolingual Language Models for 350 Languages}, + author={Tyler A. Chang and Catherine Arnett and Zhuowen Tu and Benjamin K. Bergen}, + year={2026}, + eprint={2408.10441}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2408.10441}, +} + +@inproceedings{scaffold_bpe, + author = {Lian, Haoran and Xiong, Yizhe and Niu, Jianwei and Mo, Shasha and Su, Zhenpeng and Lin, Zijia and Chen, Hui and Han, Jungong and Ding, Guiguang}, + title = {{Scaffold-BPE}: enhancing byte pair encoding for large language models with simple and effective scaffold token removal}, + year = {2025}, + isbn = {978-1-57735-897-8}, + publisher = {AAAI Press}, + url = {https://doi.org/10.1609/aaai.v39i23.34633}, + doi = {10.1609/aaai.v39i23.34633}, + booktitle = {Proceedings of the Thirty-Ninth AAAI Conference on Artificial Intelligence and Thirty-Seventh Conference on Innovative Applications of Artificial Intelligence and Fifteenth Symposium on Educational Advances in Artificial Intelligence}, + articleno = {2735}, + numpages = {10}, + series = {AAAI'25/IAAI'25/EAAI'25} +} + +@inproceedings{sennrich-etal-2017-university, + title = "The {U}niversity of {E}dinburgh{'}s Neural {MT} Systems for {WMT}17", + author = "Sennrich, Rico and + Birch, Alexandra and + Currey, Anna and + Germann, Ulrich and + Haddow, Barry and + Heafield, Kenneth and + Miceli Barone, Antonio Valerio and + Williams, Philip", + editor = "Bojar, Ond{\v{r}}ej and + Buck, Christian and + Chatterjee, Rajen and + Federmann, Christian and + Graham, Yvette and + Haddow, Barry and + Huck, Matthias and + Yepes, Antonio Jimeno and + Koehn, Philipp and + Kreutzer, Julia", + booktitle = "Proceedings of the Second Conference on Machine Translation", + month = sep, + year = "2017", + address = "Copenhagen, Denmark", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W17-4739/", + doi = "10.18653/v1/W17-4739", + pages = "389--399" +} + +@inproceedings{schmidt2025boundless, + title={Boundless Byte Pair Encoding: Breaking the Pre-tokenization Barrier}, + author={Craig W. Schmidt and Varshini Reddy and Chris Tanner and Yuval Pinter}, + booktitle={Second Conference on Language Modeling}, + year={2025}, + url={https://openreview.net/forum?id=oPAjXGV8qQ} +} + +@inproceedings{galle-2019-investigating, + title = "Investigating the Effectiveness of {BPE}: The Power of Shorter Sequences", + author = "Gall{\'e}, Matthias", + editor = "Inui, Kentaro and + Jiang, Jing and + Ng, Vincent and + Wan, Xiaojun", + booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)", + month = nov, + year = "2019", + address = "Hong Kong, China", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/D19-1141/", + doi = "10.18653/v1/D19-1141", + pages = "1375--1381" +} + +@misc{schmidt2026tokenizationsplittrees, + title={Tokenization with Split Trees}, + author={Craig W. Schmidt and Michael Krumdick and Adam Wiemerslage and Seth Ebner and Varshini Reddy and Yuval Pinter and Chris Tanner}, + year={2026}, + eprint={2605.22705}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2605.22705}, + note = {(Withdrawn)} +} + +@misc{diao2025nemotronclimbclusteringbasediterativedata, + title={Nemotron-{CLIMB}: CLustering-based Iterative Data Mixture Bootstrapping for Language Model Pre-training}, + author={Shizhe Diao and Yu Yang and Yonggan Fu and Xin Dong and Dan Su and Markus Kliegl and Zijia Chen and Peter Belcak and Yoshi Suhara and Hongxu Yin and Mostofa Patwary and Yingyan Lin and Jan Kautz and Pavlo Molchanov}, + year={2025}, + eprint={2504.13161}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2504.13161}, +} + +@misc{nanochat, + author = {Andrej Karpathy}, + title = {nanochat: The best {ChatGPT} that \$100 can buy}, + year = {2025}, + publisher = {GitHub}, + url = {https://github.com/karpathy/nanochat} +} + +@inproceedings{li2024datacomplm, + author = {Li, Jeffrey and Fang, Alex and Smyrnis, Georgios and Ivgi, Maor and Jordan, Matt and Gadre, Samir and Bansal, Hritik and Guha, Etash and Keh, Sedrick and Arora, Kushal and Garg, Saurabh and Xin, Rui and Muennighoff, Niklas and Heckel, Reinhard and Mercat, Jean and Chen, Mayee and Gururangan, Suchin and Wortsman, Mitchell and Albalak, Alon and Bitton, Yonatan and Nezhurina, Marianna and Abbas, Amro and Hsieh, Cheng-Yu and Ghosh, Dhruba and Gardner, Josh and Kilian, Maciej and Zhang, Hanlin and Shao, Rulin and Pratt, Sarah and Sanyal, Sunny and Ilharco, Gabriel and Daras, Giannis and Marathe, Kalyani and Gokaslan, Aaron and Zhang, Jieyu and Chandu, Khyathi and Nguyen, Thao and Vasiljevic, Igor and Kakade, Sham and Song, Shuran and Sanghavi, Sujay and Faghri, Fartash and Oh, Sewoong and Zettlemoyer, Luke and Lo, Kyle and El-Nouby, Alaaeldin and Pouransari, Hadi and Toshev, Alexander and Wang, Stephanie and Groeneveld, Dirk and Soldaini, Luca and Koh, Pang Wei and Jitsev, Jenia and Kollar, Thomas and Dimakis, Alexandros G. and Carmon, Yair and Dave, Achal and Schmidt, Ludwig and Shankar, Vaishaal}, + booktitle = {Advances in Neural Information Processing Systems}, + doi = {10.52202/079017-0455}, + pages = {14200--14282}, + publisher = {Curran Associates, Inc.}, + title = {{DataComp-LM}: In search of the next generation of training sets for language models}, + url = {https://proceedings.neurips.cc/paper_files/paper/2024/file/19e4ea30dded58259665db375885e412-Paper-Datasets_and_Benchmarks_Track.pdf}, + volume = {37}, + year = {2024} +} + +% --------------------------------------------------------------------------------------- +% NOT IN THE SUPPLIED BIBLIOGRAPHY -- added because this paper cites them. +% Replace or remove if canonical entries exist. + +% The MinGram trainer used for the second half of the experiments. Title, author and date +% verified against arXiv in this session. +@misc{land2026mingram, + title={{MinGram}: A Minimalist Unigram Tokenizer with High Compression and Competitive Morphological Alignment}, + author={Sander Land}, + year={2026}, + eprint={2606.27019}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2606.27019}, +} + +% Cited for the observation that a production tokenizer uses separate opening/closing +% word markers with a minimum-piece segmenter. AUTHOR/YEAR UNVERIFIED - same Substack as +% tokencontributions-gpt4, please confirm. +@misc{tokencontributions-claude, + title={On the Biology of {C}laude's Tokenizer}, + author={Sander Land}, + year={2026}, + url = {https://tokencontributions.substack.com/p/on-the-biology-of-claudes-tokenizer}, + howpublished = "Blog Post" +} + +% Web corpus for the English-prose and mixed-domain experiments. The supplied +% bibliography contains FineWiki but not FineWeb. UNVERIFIED. +@misc{penedo2024fineweb, + author = {Guilherme Penedo and Hynek Kydl{\'i}{\v{c}}ek and Loubna Ben Allal and Anton Lozhkov and Margaret Mitchell and Colin Raffel and Leandro Von Werra and Thomas Wolf}, + title = {The {FineWeb} Datasets: Decanting the Web for the Finest Text Data at Scale}, + year = {2024}, + eprint = {2406.17557}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + url = {https://arxiv.org/abs/2406.17557}, + note = {NOT in the supplied bibliography; details unverified} +} + +% Code corpora for the mixed prose+code experiment. UNVERIFIED. +@misc{codeparrot, + title = {{CodeParrot} Clean}, + author = {{CodeParrot}}, + howpublished = {\url{https://huggingface.co/datasets/codeparrot/codeparrot-clean}}, + year = {2022}, + note = {NOT in the supplied bibliography; details unverified} +} + +@misc{rosettacode, + title = {Rosetta Code}, + howpublished = {\url{https://huggingface.co/datasets/christopher/rosetta-code}}, + year = {2023}, + note = {NOT in the supplied bibliography; details unverified} +} diff --git a/marker_experiments/prior_results.json b/marker_experiments/prior_results.json new file mode 100644 index 00000000..5e50be56 --- /dev/null +++ b/marker_experiments/prior_results.json @@ -0,0 +1,156 @@ +{ + "_note": "Results measured earlier in this investigation. The scratchpad holding the original result JSONs was wiped twice by the environment; these values are transcribed from the run logs of those sessions. The FineWiki 6-language results live in multilang_result.json, produced by multilang_grid.py.", + "_common": { + "pretokenizer_base": "ScriptEncodingV3, enforce_char_boundaries=True (registry name scriptenc3_cb)", + "metric": "chars per token on held-out documents, higher is better", + "mingram_overshoot_factor": 1.15, + "num_workers": 4 + }, + + "fineweb_prose": { + "setup": "HuggingFaceFW/fineweb sample-10BT, 80M chars train (26,351 docs), 1000 held-out docs (3,104,566 chars), no whitespace normalization", + "chars_per_token": { + "plain_bpe": {"16k": 3.9909, "32k": 4.2611, "64k": 4.4412}, + "v4_bpe": {"16k": 3.9266, "32k": 4.2170, "64k": 4.4032}, + "plain_mingram":{"16k": 4.0474, "32k": 4.3012, "64k": 4.4678}, + "v4_mingram": {"16k": 3.9670, "32k": 4.2483, "64k": 4.4204} + }, + "gap_vs_plain_pct": { + "bpe": {"16k": -1.61, "32k": -1.03, "64k": -0.86}, + "mingram": {"16k": -1.99, "32k": -1.23, "64k": -1.06} + }, + "space_dup_pairs": { + "plain_bpe": {"16k": 1792, "32k": 4201, "64k": 9871}, + "v4_bpe": {"16k": 1, "32k": 2, "64k": 2} + }, + "space_dup_vocab_frac_pct": { + "plain_bpe": {"16k": 20.24, "32k": 24.92, "64k": 30.04}, + "plain_mingram": {"16k": 21.31, "32k": 25.84, "64k": 31.51}, + "v4": {"16k": 0.01, "32k": 0.01, "64k": 0.01} + }, + "marker_variant_extra_slots_v4": {"bpe": {"16k": 115, "32k": 191, "64k": 341}, + "mingram": {"16k": 121, "32k": 199, "64k": 355}}, + "distinct_alpha_words_with_own_token": { + "plain_bpe": {"16k": 13836, "32k": 27011, "64k": 52355}, + "v4_bpe": {"16k": 7717, "32k": 17296, "64k": 37401}, + "plain_mingram": {"16k": 13713, "32k": 26741, "64k": 51695}, + "v4_mingram": {"16k": 8820, "32k": 19598, "64k": 42395} + }, + "unique_chunks": {"plain": 334837, "v4": 277862, "v3_fused": 1764948}, + "bpe_train_seconds": {"plain": {"16k": 63, "32k": 71, "64k": 107}, + "v4": {"16k": 60, "32k": 65, "64k": 139}}, + "mingram_train_seconds": {"plain": {"16k": 127, "32k": 157, "64k": 237}, + "v4": {"16k": 107, "32k": 143, "64k": 227}}, + "roundtrip_failures": "0 for every cell" + }, + + "earlier_variants_prose_16k": { + "_note": "Design iterations at 16k on the 80M-char FineWeb corpus, BPE.", + "plain": 3.9909, + "v1_rename_merged_space": 3.9909, + "v2_hard_chunk_boundary_no_elision": 2.4201, + "v3_words_only_elided_mergeable": 3.6815, + "v3_words_only_plus_merge_ban": 3.6920, + "v3_dup_pairs": 45, + "v3_unique_chunks": 1764948, + "v3_bpe_train_seconds": 442 + }, + + "mingram_overshoot_sweep_prose_16k": { + "_note": "Does a larger BPE-init overshoot let MinGram buy back the marker penalty? No.", + "plain": {"1.10": 4.0455, "1.15": 4.0474, "1.25": 4.0470}, + "v3": {"1.10": 3.7286, "1.15": 3.7282, "1.25": 3.7252}, + "v3_word_coverage": {"1.10": 8674, "1.15": 8856, "1.25": 8964}, + "plain_word_coverage": {"1.10": 13734, "1.15": 13713, "1.25": 13674} + }, + + "mixed_prose_code": { + "setup": "40.0M chars FineWeb prose + 40.0M chars code (codeparrot-clean-valid Python files + christopher/rosetta-code multi-language snippets); 500 held-out docs per domain; no whitespace normalization", + "chars_per_token": { + "plain_bpe": {"16k": {"mixed": 3.5810, "prose": 3.8479, "code": 2.7715}, + "32k": {"mixed": 3.8295, "prose": 4.1471, "code": 2.8956}, + "64k": {"mixed": 4.0087, "prose": 4.3546, "code": 3.0039}}, + "v4_bpe": {"16k": {"mixed": 3.5046, "prose": 3.7862, "code": 2.6685}, + "32k": {"mixed": 3.7669, "prose": 4.0988, "code": 2.8090}, + "64k": {"mixed": 3.9554, "prose": 4.3111, "code": 2.9355}}, + "v5_bpe": {"16k": {"mixed": 3.5774, "prose": 3.8500, "code": 2.7558}, + "32k": {"mixed": 3.8552, "prose": 4.1798, "code": 2.9051}, + "64k": {"mixed": 4.0557, "prose": 4.4040, "code": 3.0425}}, + "plain_mingram": {"16k": {"mixed": 3.6310, "prose": 3.9085, "code": 2.7951}, + "32k": {"mixed": 3.8721, "prose": 4.1937, "code": 2.9269}, + "64k": {"mixed": 4.0363, "prose": 4.3885, "code": 3.0167}}, + "v4_mingram": {"16k": {"mixed": 3.5438, "prose": 3.8304, "code": 2.6946}, + "32k": {"mixed": 3.7994, "prose": 4.1350, "code": 2.8317}, + "64k": {"mixed": 3.9780, "prose": 4.3376, "code": 2.9485}}, + "v5_mingram": {"16k": {"mixed": 3.6180, "prose": 3.8962, "code": 2.7817}, + "32k": {"mixed": 3.8884, "prose": 4.2166, "code": 2.9284}, + "64k": {"mixed": 4.0795, "prose": 4.4315, "code": 3.0569}} + }, + "gap_vs_plain_pct": { + "v4_bpe": {"16k": {"mixed": -2.13, "prose": -1.60, "code": -3.71}, + "32k": {"mixed": -1.64, "prose": -1.17, "code": -2.99}, + "64k": {"mixed": -1.33, "prose": -1.00, "code": -2.28}}, + "v5_bpe": {"16k": {"mixed": -0.10, "prose": 0.05, "code": -0.56}, + "32k": {"mixed": 0.67, "prose": 0.79, "code": 0.33}, + "64k": {"mixed": 1.17, "prose": 1.13, "code": 1.29}}, + "v4_mingram": {"16k": {"mixed": -2.40, "prose": -2.00, "code": -3.60}, + "32k": {"mixed": -1.88, "prose": -1.40, "code": -3.25}, + "64k": {"mixed": -1.44, "prose": -1.16, "code": -2.26}}, + "v5_mingram": {"16k": {"mixed": -0.36, "prose": -0.32, "code": -0.48}, + "32k": {"mixed": 0.42, "prose": 0.55, "code": 0.05}, + "64k": {"mixed": 1.07, "prose": 0.98, "code": 1.33}} + }, + "space_dup_pairs": { + "plain_bpe": {"16k": 3364, "32k": 6887, "64k": 13480}, + "plain_mingram": {"16k": 3506, "32k": 7139, "64k": 14038}, + "v4_bpe": {"16k": 36, "32k": 67, "64k": 120}, + "v5_bpe": {"16k": 36, "32k": 66, "64k": 120}, + "v5_mingram": {"16k": 40, "32k": 77, "64k": 129} + }, + "space_dup_vocab_frac_pct": { + "plain_bpe": {"16k": 38.0, "32k": 40.9, "64k": 41.0}, + "plain_mingram": {"16k": 39.6, "32k": 42.4, "64k": 42.7}, + "v4_or_v5": {"16k": 0.4, "32k": 0.4, "64k": 0.4} + }, + "marker_variant_extra_slots_v5": {"bpe": {"16k": 310, "32k": 520, "64k": 993}, + "mingram": {"16k": 343, "32k": 570, "64k": 1084}}, + "bpe_train_seconds_64k": {"plain": 108, "v5": 92}, + "mingram_train_seconds_64k": {"plain": 222, "v5": 191}, + "roundtrip_failures": "1 for every cell, plain included: U+F8FF (private use) is absent from the V3 char_encoding and is dropped. Pre-existing, not marker-related." + }, + + "diagnostics": { + "space_run_profile_1gb_scans": { + "fineweb_en": {"chars": 1000001224, "docs": 326173, "single_spaces": 163068065, + "multi_space_runs": 3940, + "left_neighbour_pct": {"lower": 86.26, "punct": 10.67, "upper": 1.81, "digit": 1.15, "symbol": 0.10}, + "right_neighbour_pct": {"lower": 82.07, "upper": 14.49, "digit": 1.69, "punct": 1.56, "symbol": 0.17}}, + "finewiki_en_raw": {"chars": 1000003320, "docs": 189736, "single_spaces": 113157392, + "multi_space_runs": 11749474, "chars_in_runs": 275771070, "max_run": 112665, + "_note": "raw text field retains wikitable markup; '|' is the #2 left neighbour at 12.9M. The registry's finewiki loader applies normalize_whitespace, which removes these runs."} + }, + "domain_space_profile": { + "prose_sample": {"chars": 583155, "single_space_pct_of_chars": 16.09, "multi_runs": 3}, + "code_sample": {"chars": 1789198, "single_space_pct_of_chars": 5.92, "multi_runs": 33221, + "pct_chars_in_runs": 16.77, "modal_run_lengths": [8, 4, 12, 16]} + }, + "indentation_token_cost_code_eval": { + "_note": "Multi-space indentation is ~1 token either way and cannot explain the code gap.", + "pure_space_tokens_len_gt_1": 1717, "pct_of_all_code_tokens": 1.24, + "identical_under_plain_and_v4": true, + "whitespace_bucket_chars_per_token": {"plain": 4.59, "v4": 4.30} + }, + "token_delta_v4_minus_plain_code_eval": { + "total": 3256, "whitespace": 1708, "alpha": 721, "punct": 434, "marker_only": 382, "digit": 11 + }, + "non_elided_single_spaces": { + "code": {"v4_pct": 12.5, "v5_pct": 0.3, "digit_adjacent_share_of_non_elided_pct": 97.9}, + "prose": {"v4_pct": 3.3, "v5_pct": 0.0, "digit_adjacent_share_of_non_elided_pct": 98.7} + }, + "duplicate_token_mass_prose_16k_plain": { + "pairs": 1792, "vocab_slots": 3584, "vocab_frac_pct": 22.4, + "share_of_emitted_tokens_pct": 64.23, + "top_pairs": [[" .", ".", 700931], [" ,", ",", 680504], [" the", "the", 639584]] + } + } +} diff --git a/marker_experiments/scriptenc_marker_v4.py b/marker_experiments/scriptenc_marker_v4.py new file mode 100644 index 00000000..63d8ff48 --- /dev/null +++ b/marker_experiments/scriptenc_marker_v4.py @@ -0,0 +1,190 @@ +"""Boundary-marker pretokenizer, v4: markers on word spans AND punctuation. + +Design +------ + * word spans (LM scripts that use spaces) get <|> on BOTH sides, always -- + that is the point: 'the' looks identical whether or not a space preceded it, + which kills the ' the'/'the' duplicate vocabulary pair. + * punctuation gets <|> ONLY on a side that actually had a single space, which + was then elided. So none in 'a=b', one in 'a, b', two in 'a = b'. + +The asymmetry is what keeps decoding unambiguous. If punctuation were wrapped +unconditionally, 'a,b' would give <|>a<|> <|>,<|> <|>b<|> -- touching markers at +both junctions, indistinguishable from 'a , b'. Marking punctuation only on a +space side preserves the invariant: + + two <|> touching <=> exactly one elided space + a lone <|> <=> pure structural boundary, contributes no character + +Word-adjacency exception +------------------------ +Two *different* word scripts directly adjacent with no space (Greek letters used +as identifiers: 'upperDelta', 'spi', 'Deltax') would both emit unconditional +markers, they would touch, and decode would insert a phantom space. This is not +rare once code is in the mix -- it hit 5/500 held-out code documents. A word +therefore drops its OPENING marker when the preceding unit is also a word, which +(since _build_units absorbs same-script and inherited runs) can only mean a +script change with no space. The preceding word keeps its closing marker, which +is then a lone marker and decodes to nothing. + +Chunking +-------- +Units are NOT fused across an elided space. Since bpe_merge_allowed already +forbids merging across two touching markers, fusing yields exactly the same set +of legal merges while inflating the corpus ~5x in unique chunks and BPE training +~10x in wall clock. Decode reads the flat atomic stream, so chunk boundaries do +not affect reconstruction -- markers still touch across a boundary. +""" + +import itertools +from typing import Sequence + +from script_bpe.pretokenize.pretokenizer import ScriptPretokenizer, ScriptPretokenizerConfig, CharEncT + + +class MarkerCharEnc: + __slots__ = ("script_id", "combines_with_spaces", "atomic_token_ids", "inherited") + + def __init__(self, token_id: int): + self.script_id = -2 # sentinel, never scanned via groupby -- inserted directly + self.combines_with_spaces = False + self.inherited = False + self.atomic_token_ids = [token_id] + + def __repr__(self): + return f"MarkerCharEnc(atomic_token_ids={self.atomic_token_ids})" + + +class MarkerV4PretokenizerConfig(ScriptPretokenizerConfig): + cls: str = "MarkerV4Pretokenizer" + + +class MarkerV4Pretokenizer(ScriptPretokenizer, config_type=MarkerV4PretokenizerConfig): + MARKER_TEXT = "<|>" + + def _build_atomic_tokens(self): + super()._build_atomic_tokens() + self.marker_token_id = self._register_token(self.MARKER_TEXT) + self.is_initial_char_tokens.add(self.marker_token_id) + # word scripts: category LM (letters) AND combines_with_spaces. Excludes + # Han/Hiragana/Katakana/Thai (LM but spaceless) and punctuation (which is + # combines_with_spaces under V3 via the (ALL, "PSF") entry, but not LM). + self.lm_wrap_script_ids = { + block.script_id + for block in self.config.script_config.blocks + if block.category == "LM" and block.combines_with_spaces + } + # Subclass hook (see v5): extra script ids to treat like punctuation -- markable + # on a side whose single space was elided, never marked unconditionally. + self.extra_markable_script_ids = self._extra_markable_script_ids() + + def _extra_markable_script_ids(self) -> set: + return set() + + def bpe_merge_allowed(self, a, b) -> bool: + # No learned token may span an elided-space point. With per-unit chunks this + # junction never falls inside a chunk, so it is belt-and-braces -- but it also + # guarantees no token can contain '<|><|>', which would reintroduce per-word + # duplication keyed on what follows instead of what precedes. + if a[-1] == self.marker_token_id and b[0] == self.marker_token_id: + return False + return super().bpe_merge_allowed(a, b) + + def decode(self, tokenization, errors="replace") -> str: + decoded = "" + i = 0 + n = len(tokenization) + while i < n: + if tokenization[i] == self.marker_token_id: + if i + 1 < n and tokenization[i + 1] == self.marker_token_id: + decoded += " " # two markers touching = one elided space + i += 2 + else: + i += 1 # lone marker: structural boundary only + continue + script_tok = tokenization[i] + ix_tok = tokenization[i + 1] if i + 1 < n else None + if (script_tok, ix_tok) in self.detokenize_map: + decoded += self.detokenize_map[(script_tok, ix_tok)] + i += 2 + else: + if errors == "backslashreplace": + decoded += self.atomic_tokens[script_tok] + elif errors == "replace": + decoded += "�" + elif errors == "strict": + raise ValueError(f"Invalid tokenization: ({script_tok}, {ix_tok}) is not a valid token pair!") + else: + raise ValueError(f"Unknown error handling mode: {errors}") + i += 1 + return decoded + + # unit kinds + WORD, PUNCT, SPACE, OTHER = "word", "punct", "space", "other" + + def _build_units(self, script_groups) -> list[tuple[str, list]]: + """Collapse script groups into units, absorbing inherited/same-script continuations.""" + lm_ids = self.lm_wrap_script_ids + space_group = self.space_group + units: list[tuple[str, list]] = [] + i = 0 + while i < len(script_groups): + group = script_groups[i] + if group == space_group: # exactly one space character + units.append((self.SPACE, list(group))) + i += 1 + continue + script_id = group[0].script_id + if script_id in lm_ids: + kind = self.WORD + elif group[0].combines_with_spaces or script_id in self.extra_markable_script_ids: + kind = self.PUNCT + else: + kind = self.OTHER + content = list(group) + i += 1 + while i < len(script_groups) and ( + script_groups[i][0].inherited or script_groups[i][0].script_id == script_id + ): + content += script_groups[i] + i += 1 + units.append((kind, content)) + return units + + def split_encoded(self, encoding: Sequence[CharEncT]) -> list[Sequence[CharEncT]]: + if not self._script_split: + return [encoding] + script_encoding = [c for c in encoding if hasattr(c, "script_id")] + if len(script_encoding) != len(encoding): + raise ValueError(f"Unexpected encoding: {encoding}") + script_groups = [list(g) for _, g in itertools.groupby(script_encoding, key=lambda x: x.script_id)] + units = self._build_units(script_groups) + marker = MarkerCharEnc(self.marker_token_id) + + # A single space is elided when both neighbours are markable (word or punct); the + # facing sides then carry a marker, so the two markers touch in the atomic stream. + markable = (self.WORD, self.PUNCT) + elided = [False] * len(units) + for i, (kind, _) in enumerate(units): + if kind != self.SPACE: + continue + if 0 < i < len(units) - 1 and units[i - 1][0] in markable and units[i + 1][0] in markable: + elided[i] = True + + chunks: list[Sequence[CharEncT]] = [] + for i, (kind, content) in enumerate(units): + if kind == self.SPACE: + if not elided[i]: + chunks.append(content) # left untouched, exactly as the baseline encodes it + continue + if kind == self.OTHER: + chunks.append(content) + continue + if kind == self.WORD: + right = True + left = not (i > 0 and units[i - 1][0] == self.WORD) # see word-adjacency note + else: # PUNCT: only on a side whose space was actually elided + left = i > 0 and elided[i - 1] + right = i + 1 < len(units) and elided[i + 1] + chunks.append(([marker] if left else []) + content + ([marker] if right else [])) + return chunks diff --git a/marker_experiments/scriptenc_marker_v5.py b/marker_experiments/scriptenc_marker_v5.py new file mode 100644 index 00000000..ffe84efc --- /dev/null +++ b/marker_experiments/scriptenc_marker_v5.py @@ -0,0 +1,49 @@ +"""Boundary-marker pretokenizer, v5 = v4 + digits are markable. + +Measured motivation (held-out prose and code evals, under v4): + + domain single spaces elided not elided of which digit-adjacent + code 42,242 87.5% 12.5% 97.9% + prose 287,220 96.7% 3.3% 98.7% + +Digits account for ~98% of every single space v4 fails to elide, and code has +3.8x prose's rate of them -- which is why v4's code penalty ran ~2x its prose +penalty. It was never indentation: multi-space runs are ~1.2% of emitted tokens +and identical under the baseline and v4. + +Why digits were excluded to begin with: script_category_v3 folds L/M -> LM, +Z/Cc -> ZC, So -> So and P/S/Cf -> PSF, but leaves category N alone. Digits +therefore land in (ALL, "N") blocks, which are absent from script_cat_with_spaces +and so were classified OTHER (non-markable) by v4. + +v5 marks digits with the SAME asymmetric rule as punctuation -- a marker only on +a side whose single space was actually elided, never unconditionally: + + 'x = 1' -> <|>x<|> <|>=<|> <|>1 (both spaces elided) + 'a1' -> <|>a<|> 1 (no space, no marker) + '1a' -> 1 <|>a<|> (no space, no marker) + +The invariant survives -- two touching <|> still means exactly one elided space -- +because only word spans are ever marked unconditionally, and word|word adjacency +is already handled in v4. + +Digits are a closed set (10 per script), so variant cost stays bounded, as it +does for punctuation. Deliberately NOT extended to Han/emoji/other scripts: +those are open sets where per-unit marker variants could multiply, and they +account for only ~2% of non-elided spaces. +""" + +from script_bpe.pretokenize.pretokenizer import ScriptPretokenizerConfig + +from scriptenc_marker_v4 import MarkerV4Pretokenizer + + +class MarkerV5PretokenizerConfig(ScriptPretokenizerConfig): + cls: str = "MarkerV5Pretokenizer" + + +class MarkerV5Pretokenizer(MarkerV4Pretokenizer, config_type=MarkerV5PretokenizerConfig): + def _extra_markable_script_ids(self) -> set: + # category "N" (Nd/Nl/No) survives script_category_v3's supercategory folding + # untouched, with script rewritten to ALL as for all non-letters. + return {b.script_id for b in self.config.script_config.blocks if b.category == "N"} diff --git a/marker_experiments/test_boundary.py b/marker_experiments/test_boundary.py new file mode 100644 index 00000000..752ea6c3 --- /dev/null +++ b/marker_experiments/test_boundary.py @@ -0,0 +1,432 @@ +"""Tests for boundary-marker pretokenization. + +Run: .venv/bin/python -m pytest marker_experiments/test_boundary.py -q +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from script_bpe.pretokenize import get_pretokenizer +from script_bpe.pretokenize.scriptencoding import ScriptEncodingV3 + +from boundary_pretokenizer import ( + BOUNDARY_VARIANTS, + BoundaryScriptPretokenizer, + BoundaryScriptPretokenizerConfig, + get_boundary_pretokenizer, +) + +VARIANTS = list(BOUNDARY_VARIANTS) + +TEXTS = [ + "the quick, brown dog", + "a, b", "a=b", "a = b", "a,b", "a ,b", "a , b", + "x = 1", "1 item", "range(0, 10)", "a1", "1a", "3.14 and 2", "version 2 of 3", + " double spaces here", " leading space", "trailing space ", "", " ", " ", + "x\ny", "tab\there", + "e.g. 3.14 and (1 + 2) = 3", + 'quote: "a b" done', + # cross-script word spans, no separator + "latinкириллица", "кириллицаlatin", "123 latinкириллица 123", + "sπ = 1", "upperΔ = 1", ">>> Δx = 1", "łʒλπ", "aΔб", "abcΔ def", + # non-space scripts and other categories + "中文 and English 42", "emoji 🎉 7 test", "日本語テキスト", + "русский текст, да", "العربية نص", "한국어 텍스트", + "def f():\n if x > 1:\n return 2\n", + "naïve café résumé", "file", "élan", # combining marks / inherited +] + + +def visible(pt, ids): + """Render marker tokens literally; decode() would turn them into spaces or nothing.""" + out, i = "", 0 + while i < len(ids): + if ids[i] == pt.marker_token_id: + out += "<|>" + i += 1 + else: + out += pt.tokens_repr(ids[i : i + 2]) + i += 2 + return out + + +def flat(pt, text): + return [t for c in pt.pretokenize(text) for t in c] + + +@pytest.fixture(scope="module", params=VARIANTS) +def pt(request): + return get_boundary_pretokenizer(request.param) + + +# --------------------------------------------------------------------------- roundtrip + + +@pytest.mark.parametrize("text", TEXTS) +def test_roundtrip(pt, text): + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +def test_roundtrip_taylorswift(pt): + with open("tests/data/taylorswift.txt") as f: + text = f.read() + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +# ------------------------------------------------------------------- the core invariant + + +@pytest.mark.parametrize("text", TEXTS) +def test_touching_markers_mean_exactly_one_space(pt, text): + """Every <|><|> pair must correspond to a real single space in the source, and the + count of pairs must equal the number of spaces the decoder reinserts.""" + ids = flat(pt, text) + m = pt.marker_token_id + pairs, i = 0, 0 + while i < len(ids): + if ids[i] == m and i + 1 < len(ids) and ids[i + 1] == m: + pairs += 1 + i += 2 + else: + i += 1 + # decoding without marker handling drops elided spaces; the difference is the pair count + n_spaces_source = pt.normalize(text).count(" ") + n_spaces_kept = sum( + 1 + for c in pt.pretokenize(text) + for t in c + if t != m + ) + assert pairs <= n_spaces_source, "more elided spaces claimed than exist" + assert pt.decode(ids) == pt.normalize(text) + assert n_spaces_kept >= 0 + + +@pytest.mark.parametrize("text", TEXTS) +def test_no_triple_marker_run(pt, text): + """Three markers in a row would be ambiguous (one space plus a lone marker, or the + reverse). Span merging plus asymmetric punctuation marking must prevent it.""" + ids = flat(pt, text) + m = pt.marker_token_id + for i in range(len(ids) - 2): + assert not (ids[i] == m and ids[i + 1] == m and ids[i + 2] == m), f"triple marker in {text!r}" + + +# -------------------------------------------------------------- span merging (no middle marker) + + +def test_cross_script_word_span_has_no_internal_marker(): + """The whole point of merging spans: latin+Cyrillic with no space is ONE span, so no + marker appears between them and no phantom space can be decoded.""" + for name in VARIANTS: + pt = get_boundary_pretokenizer(name) + ids = flat(pt, "latinкириллица") + m = pt.marker_token_id + assert ids[0] == m and ids[-1] == m, f"{name}: span not delimited" + assert ids.count(m) == 2, f"{name}: expected exactly 2 markers, got {ids.count(m)}" + assert pt.decode(ids) == "latinкириллица" + + +def test_cross_script_span_keeps_canonical_form_for_both_scripts(): + """A word keeps the same delimited form whether or not a different-script word + precedes it, because the span is delimited as a whole.""" + pt = get_boundary_pretokenizer("bnd_wpd") + m = pt.marker_token_id + # 'кириллица' alone vs after a Latin run: in both cases the SPAN carries the markers + alone = flat(pt, "кириллица") + assert alone[0] == m and alone[-1] == m and alone.count(m) == 2 + + +def test_word_span_always_delimited_both_sides(): + pt = get_boundary_pretokenizer("bnd_wpd") + m = pt.marker_token_id + for text in ["word", "(word", "word)", "1word", "word1", ",word,", "=word="]: + ids = flat(pt, text) + # locate the word span's markers: there must be a marker immediately before the + # first letter token and after the last + assert ids.count(m) >= 2, f"{text!r} -> {visible(pt, ids)}" + assert pt.decode(ids) == text + + +def test_internal_script_split_preserved(): + """Markers ride the outer chunks; the script boundary inside the span still splits + chunks, so no BPE merge can cross a script change the baseline would forbid.""" + pt = get_boundary_pretokenizer("bnd_wpd") + chunks = pt.pretokenize("latinкириллица") + assert len(chunks) == 2, [visible(pt, list(c)) for c in chunks] + assert visible(pt, list(chunks[0])).startswith("<|>") + assert visible(pt, list(chunks[1])).endswith("<|>") + + +# ----------------------------------------------------------------- boundary target behaviour + + +def test_punct_only_marked_on_space_side(): + pt = get_boundary_pretokenizer("bnd_wp") + assert visible_chunks(pt, "a,b") == ["<|>a<|>", ",", "<|>b<|>"] + assert visible_chunks(pt, "a, b") == ["<|>a<|>", ",<|>", "<|>b<|>"] + assert visible_chunks(pt, "a ,b") == ["<|>a<|>", "<|>,", "<|>b<|>"] + assert visible_chunks(pt, "a = b") == ["<|>a<|>", "<|>=<|>", "<|>b<|>"] + + +def test_digits_marked_only_in_wpd(): + wp = get_boundary_pretokenizer("bnd_wp") + wpd = get_boundary_pretokenizer("bnd_wpd") + # under bnd_wp a digit is not markable, so the space cannot be elided and survives + assert " " in "".join(visible_chunks(wp, "x = 1")) + # under bnd_wpd both spaces are elided + assert visible_chunks(wpd, "x = 1") == ["<|>x<|>", "<|>=<|>", "<|>1"] + + +def test_word_only_variant_leaves_punct_and_digits_bare(): + pt = get_boundary_pretokenizer("bnd_w") + assert visible_chunks(pt, "a, b") == ["<|>a<|>", ",", " ", "<|>b<|>"] + assert visible_chunks(pt, "a b") == ["<|>a<|>", "<|>b<|>"] + + +def test_multi_space_runs_untouched(pt): + """Only a single space is ever elided; runs are left exactly as the baseline emits.""" + for text in ["a b", "a b", "a\t b"]: + assert pt.decode(flat(pt, text)) == pt.normalize(text) + assert " " in "".join(visible_chunks(pt, "a b")) + + +def test_leading_and_trailing_space_not_elided(pt): + for text in [" a", "a "]: + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +# ------------------------------------------------------------------------ merge constraint + + +def test_merge_across_touching_markers_forbidden(pt): + m = pt.marker_token_id + assert pt.bpe_merge_allowed([m], [m]) is False + assert pt.bpe_merge_allowed([1, m], [m, 2]) is False + + +def test_lone_marker_merge_allowed(pt): + m = pt.marker_token_id + # a marker may merge with real content, that is how '<|>the<|>' is learned + letter = next(t for t in pt.atomic_tokens if t != m) + assert pt.bpe_merge_allowed([m], [letter]) is not False + + +# ------------------------------------------------------------------------------- config + + +def test_unknown_boundary_target_rejected(): + with pytest.raises(ValueError): + BoundaryScriptPretokenizer( + BoundaryScriptPretokenizerConfig(script_config=ScriptEncodingV3, boundary_targets=("word", "bogus")) + ) + + +def test_variants_have_distinct_hashes(): + """hash() is config-derived; distinct boundary_targets must not collide in the + pretokenized-corpus cache.""" + hashes = {name: get_boundary_pretokenizer(name).hash() for name in VARIANTS} + assert len(set(hashes.values())) == len(VARIANTS), hashes + assert get_pretokenizer("scriptenc3_cb").hash() not in set(hashes.values()) + + +def test_atomic_vocab_is_baseline_plus_one(): + base = len(get_pretokenizer("scriptenc3_cb").atomic_tokens) + for name in VARIANTS: + assert len(get_boundary_pretokenizer(name).atomic_tokens) == base + 1 + + +# --------------------------------------------------------------------------------- helper + + +def visible_chunks(pt, text): + return [visible(pt, list(c)) for c in pt.pretokenize(text)] + + +# ------------------------------------------------------------------- digit_handling + +DIGIT_TEXTS = [ + "x = 1", "version 2 of 3", "in 1984 he", "a 12345 b", "3.14 and 2", + "0", "007 and 42", "1a", "a1", "123 latinкириллица 123", "no digits here", +] + + +def boundary_pt(targets=("word", "punct", "digit"), digit_handling=None): + return BoundaryScriptPretokenizer( + BoundaryScriptPretokenizerConfig( + script_config=ScriptEncodingV3, boundary_targets=targets, digit_handling=digit_handling + ) + ) + + +@pytest.mark.parametrize("digit_handling", [None, "SPLIT", "RTL3"]) +@pytest.mark.parametrize("text", DIGIT_TEXTS) +def test_digit_handling_roundtrip(digit_handling, text): + pt = boundary_pt(digit_handling=digit_handling) + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +@pytest.mark.parametrize("digit_handling", ["SPLIT", "RTL3"]) +def test_elision_still_happens_across_digit_boundaries(digit_handling): + """The base pipeline splits digit runs into separate chunks BEFORE split_encoded, + which would put a digit and its neighbouring word in different chunks and silently + disable elision. The override must keep 'x = 1' fully elided.""" + pt = boundary_pt(digit_handling=digit_handling) + ids = flat(pt, "x = 1") + m = pt.marker_token_id + pairs = sum(1 for i in range(len(ids) - 1) if ids[i] == m and ids[i + 1] == m) + assert pairs == 2, f"expected both spaces elided, got {pairs}" + assert pt.decode(ids) == "x = 1" + + +@pytest.mark.parametrize("digit_handling", ["SPLIT", "RTL3"]) +def test_only_outer_digit_groups_carry_markers(digit_handling): + """Interior groups of a digit run must never be marked; that is what bounds the + number of marked digit forms (10 under SPLIT, 1110 under RTL3) instead of one set + per distinct number.""" + pt = boundary_pt(digit_handling=digit_handling) + chunks = [list(c) for c in pt.pretokenize("a 12345 b")] + m = pt.marker_token_id + digit_chunks = [c for c in chunks if any(t in pt.digit_token_ids for t in c)] + assert len(digit_chunks) >= 2, digit_chunks + for c in digit_chunks[1:-1]: + assert m not in c, f"interior digit group marked: {c}" + + +def test_split_bounds_markable_digit_strings(): + """Under SPLIT a marked digit form can only ever wrap a single digit.""" + pt = boundary_pt(digit_handling="SPLIT") + m = pt.marker_token_id + marked = set() + for text in ["a 1 b", "a 12 b", "a 12345 b", "x 007 y", "1 2 3"]: + for c in pt.pretokenize(text): + c = list(c) + if m in c: + core = [t for t in c if t != m] + if core and all(t in pt.digit_token_ids for t in core): + marked.add("".join(pt.atomic_tokens[t] for t in core)) + assert all(len(s) == 1 for s in marked), marked + + +def test_digit_target_off_leaves_digits_unmarked(): + pt = boundary_pt(targets=("word", "punct"), digit_handling="SPLIT") + m = pt.marker_token_id + for c in pt.pretokenize("x = 1"): + c = list(c) + if any(t in pt.digit_token_ids for t in c): + assert m not in c + assert pt.decode(flat(pt, "x = 1")) == "x = 1" + + +@pytest.mark.parametrize("digit_handling", [None, "SPLIT", "RTL3"]) +def test_digit_handling_changes_hash(digit_handling): + """Corpus cache keys must distinguish digit handling.""" + hashes = {dh: boundary_pt(digit_handling=dh).hash() for dh in [None, "SPLIT", "RTL3"]} + assert len(set(hashes.values())) == 3, hashes + + +NON_ASCII_NUMERIC_TEXTS = [ + "½ cup", "a ½ b", "1½", "٣ عربي", "Ⅻ century", "⅓ and 2", "½", "½½", "2½ hours", +] + + +@pytest.mark.parametrize("digit_handling", [None, "SPLIT", "RTL3"]) +@pytest.mark.parametrize("text", NON_ASCII_NUMERIC_TEXTS) +def test_non_ascii_numerics_roundtrip(digit_handling, text): + """Category N covers far more than ASCII 0-9 -- Nd for every script plus Nl/No + ('½', '⅓', '٣', 'Ⅻ') -- but group_digits/encode_digits only have tokens for ASCII, + because the base pipeline splits on re.split("([0-9]+)"). Grouping a non-ASCII + numeric raised KeyError and killed a training run mid-corpus.""" + pt = boundary_pt(digit_handling=digit_handling) + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +@pytest.mark.parametrize("digit_handling", ["SPLIT", "RTL3"]) +def test_mixed_ascii_and_non_ascii_numerics(digit_handling): + pt = boundary_pt(digit_handling=digit_handling) + for text in ["1½ cups", "٣ and 3", "12 ½ 34"]: + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +# ------------------------------------------------------------------------ caps codes + +CAPS_TEXTS = [ + "The cat", "NASA rocket", "GaN WiFi", "the cat", "A", "I am", "McDonald", + "Hello, World!", "ALL CAPS HERE", "Title Case Words", "iPhone", "eBay", "MiXeD", + "Привет Мир", "ПРИВЕТ", "Ελλάδα", "ΕΛΛΑΔΑ", "ΟΔΟΣ", "Οδος", "ές ΟΔΟΣ", + "Ünicode Ötzi", "file", "İstanbul", "STRASSE", "Straße", "ẞ", "Džungla", "DŽ", + "ARMÉE", "Ångström", "x = 1", "A1", "", +] + + +def caps_pt(caps_codes=True, digit_handling=None): + return BoundaryScriptPretokenizer( + BoundaryScriptPretokenizerConfig( + script_config=ScriptEncodingV3, boundary_targets=("word", "punct", "digit"), + caps_codes=caps_codes, digit_handling=digit_handling, + ) + ) + + +@pytest.mark.parametrize("text", CAPS_TEXTS + TEXTS) +def test_caps_roundtrip(text): + pt = caps_pt() + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +@pytest.mark.parametrize("digit_handling", ["SPLIT", "RTL3"]) +def test_caps_roundtrip_with_digit_splitting(digit_handling): + pt = caps_pt(digit_handling=digit_handling) + for text in CAPS_TEXTS + DIGIT_TEXTS: + assert pt.decode(flat(pt, text)) == pt.normalize(text) + + +def test_caps_codes_applied_where_invertible(): + pt = caps_pt() + m, sh, cp = pt.marker_token_id, pt.shift_token_id, pt.caps_token_id + assert flat(pt, "The")[1] == sh + assert flat(pt, "NASA")[1] == cp + # mixed case is left literal, as in the tokenizer this mirrors + assert sh not in flat(pt, "GaN") and cp not in flat(pt, "GaN") + assert sh not in flat(pt, "WiFi") and cp not in flat(pt, "WiFi") + assert sh not in flat(pt, "the") and cp not in flat(pt, "the") + + +def test_caps_codes_skipped_when_not_invertible(): + """Unicode case mapping is not a bijection. U+0130 lowercases to two characters and + U+1E9E uppercases to 'SS', so neither may take a caps code.""" + pt = caps_pt() + for text in ["İstanbul", "İ", "ẞ", "Džungla"]: + ids = flat(pt, text) + assert pt.shift_token_id not in ids and pt.caps_token_id not in ids, text + assert pt.decode(ids) == pt.normalize(text) + + +def test_caps_shares_the_lowercase_piece(): + """The point of the scheme: 'The' and 'the' differ by one code, not by a whole entry.""" + pt = caps_pt() + the, cap_the = flat(pt, "the"), flat(pt, "The") + assert cap_the == [cap_the[0], pt.shift_token_id] + the[1:] + + +def test_caps_codes_off_by_default(): + pt = get_boundary_pretokenizer("bnd_wpd") + assert pt.shift_token_id is None and pt.caps_token_id is None + assert len(pt.atomic_tokens) == len(caps_pt().atomic_tokens) - 2 + + +def test_caps_changes_hash(): + assert caps_pt(caps_codes=True).hash() != caps_pt(caps_codes=False).hash() + + +@pytest.mark.parametrize("text", CAPS_TEXTS) +def test_caps_preserves_no_triple_marker(text): + pt = caps_pt() + ids = flat(pt, text) + m = pt.marker_token_id + for i in range(len(ids) - 2): + assert not (ids[i] == m and ids[i + 1] == m and ids[i + 2] == m), text diff --git a/marker_experiments/tokenizers/ar_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/ar_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..22c502fb Binary files /dev/null and b/marker_experiments/tokenizers/ar_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ar_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/ar_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..294c3c2f Binary files /dev/null and b/marker_experiments/tokenizers/ar_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ar_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/ar_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..2a437dd6 Binary files /dev/null and b/marker_experiments/tokenizers/ar_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ar_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/ar_plain_bpe_32k.json.gz new file mode 100644 index 00000000..4bc8f525 Binary files /dev/null and b/marker_experiments/tokenizers/ar_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/caps250_en_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/caps250_en_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..1ed0a4d1 Binary files /dev/null and b/marker_experiments/tokenizers/caps250_en_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/caps250_en_bnd_wpd_caps_bpe_32k.json.gz b/marker_experiments/tokenizers/caps250_en_bnd_wpd_caps_bpe_32k.json.gz new file mode 100644 index 00000000..5488c0fa Binary files /dev/null and b/marker_experiments/tokenizers/caps250_en_bnd_wpd_caps_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/de_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/de_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..82139126 Binary files /dev/null and b/marker_experiments/tokenizers/de_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/de_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/de_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..1e3a45ac Binary files /dev/null and b/marker_experiments/tokenizers/de_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/de_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/de_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..494b0f31 Binary files /dev/null and b/marker_experiments/tokenizers/de_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/de_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/de_plain_bpe_32k.json.gz new file mode 100644 index 00000000..63ac0343 Binary files /dev/null and b/marker_experiments/tokenizers/de_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/en_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..8cd3e731 Binary files /dev/null and b/marker_experiments/tokenizers/en_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/en_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..857528c4 Binary files /dev/null and b/marker_experiments/tokenizers/en_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_bnd_wpd_None_bpe_32k.json.gz b/marker_experiments/tokenizers/en_bnd_wpd_None_bpe_32k.json.gz new file mode 100644 index 00000000..7de754e6 Binary files /dev/null and b/marker_experiments/tokenizers/en_bnd_wpd_None_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_bnd_wpd_RTL3_bpe_32k.json.gz b/marker_experiments/tokenizers/en_bnd_wpd_RTL3_bpe_32k.json.gz new file mode 100644 index 00000000..6aaf2417 Binary files /dev/null and b/marker_experiments/tokenizers/en_bnd_wpd_RTL3_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_bnd_wpd_SPLIT_bpe_32k.json.gz b/marker_experiments/tokenizers/en_bnd_wpd_SPLIT_bpe_32k.json.gz new file mode 100644 index 00000000..3dcc9246 Binary files /dev/null and b/marker_experiments/tokenizers/en_bnd_wpd_SPLIT_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_plain_None_bpe_32k.json.gz b/marker_experiments/tokenizers/en_plain_None_bpe_32k.json.gz new file mode 100644 index 00000000..4885bbdd Binary files /dev/null and b/marker_experiments/tokenizers/en_plain_None_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_plain_RTL3_bpe_32k.json.gz b/marker_experiments/tokenizers/en_plain_RTL3_bpe_32k.json.gz new file mode 100644 index 00000000..db95754c Binary files /dev/null and b/marker_experiments/tokenizers/en_plain_RTL3_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_plain_SPLIT_bpe_32k.json.gz b/marker_experiments/tokenizers/en_plain_SPLIT_bpe_32k.json.gz new file mode 100644 index 00000000..f081473f Binary files /dev/null and b/marker_experiments/tokenizers/en_plain_SPLIT_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/en_plain_bpe_32k.json.gz new file mode 100644 index 00000000..2df36e9a Binary files /dev/null and b/marker_experiments/tokenizers/en_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/en_plain_mingram_32k.json.gz b/marker_experiments/tokenizers/en_plain_mingram_32k.json.gz new file mode 100644 index 00000000..79a38165 Binary files /dev/null and b/marker_experiments/tokenizers/en_plain_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/fi_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/fi_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..aa62875f Binary files /dev/null and b/marker_experiments/tokenizers/fi_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/fi_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/fi_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..89e968e0 Binary files /dev/null and b/marker_experiments/tokenizers/fi_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/fi_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/fi_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..44f3163f Binary files /dev/null and b/marker_experiments/tokenizers/fi_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/fi_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/fi_plain_bpe_32k.json.gz new file mode 100644 index 00000000..ceeface2 Binary files /dev/null and b/marker_experiments/tokenizers/fi_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ko_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/ko_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..778d561b Binary files /dev/null and b/marker_experiments/tokenizers/ko_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ko_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/ko_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..ef61ec7f Binary files /dev/null and b/marker_experiments/tokenizers/ko_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ko_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/ko_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..87c4d63f Binary files /dev/null and b/marker_experiments/tokenizers/ko_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ko_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/ko_plain_bpe_32k.json.gz new file mode 100644 index 00000000..041b9fc2 Binary files /dev/null and b/marker_experiments/tokenizers/ko_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_en_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..80d623b4 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_bnd_w_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_en_bnd_w_mingram_32k.json.gz new file mode 100644 index 00000000..c2f0fbae Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_bnd_w_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_en_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..157a7148 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_bnd_wp_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_en_bnd_wp_mingram_32k.json.gz new file mode 100644 index 00000000..8d4c0e04 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_bnd_wp_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_en_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..29ca78c9 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_bnd_wpd_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_en_bnd_wpd_mingram_32k.json.gz new file mode 100644 index 00000000..c57bda89 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_bnd_wpd_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_en_plain_bpe_32k.json.gz new file mode 100644 index 00000000..c4eabced Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_en_plain_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_en_plain_mingram_32k.json.gz new file mode 100644 index 00000000..e3fe2f2f Binary files /dev/null and b/marker_experiments/tokenizers/mg250_en_plain_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..801e7e35 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_bnd_w_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_bnd_w_mingram_32k.json.gz new file mode 100644 index 00000000..9d8b458a Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_bnd_w_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..b97c500f Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_bnd_wp_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_bnd_wp_mingram_32k.json.gz new file mode 100644 index 00000000..d2c6a414 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_bnd_wp_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..b0b421c8 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_bnd_wpd_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_bnd_wpd_mingram_32k.json.gz new file mode 100644 index 00000000..fb0941b7 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_bnd_wpd_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_plain_bpe_32k.json.gz new file mode 100644 index 00000000..e5dd0593 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ko_plain_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ko_plain_mingram_32k.json.gz new file mode 100644 index 00000000..2845a41c Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ko_plain_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..a4dedf2e Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_bnd_w_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_bnd_w_mingram_32k.json.gz new file mode 100644 index 00000000..b79f3543 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_bnd_w_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..5de36c06 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_bnd_wp_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_bnd_wp_mingram_32k.json.gz new file mode 100644 index 00000000..cc9cbf51 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_bnd_wp_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..4f8a21ec Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_bnd_wpd_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_bnd_wpd_mingram_32k.json.gz new file mode 100644 index 00000000..43e44d2c Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_bnd_wpd_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_plain_bpe_32k.json.gz new file mode 100644 index 00000000..b91759c8 Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_plain_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/mg250_ru_plain_mingram_32k.json.gz b/marker_experiments/tokenizers/mg250_ru_plain_mingram_32k.json.gz new file mode 100644 index 00000000..639757ac Binary files /dev/null and b/marker_experiments/tokenizers/mg250_ru_plain_mingram_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ru_bnd_w_bpe_32k.json.gz b/marker_experiments/tokenizers/ru_bnd_w_bpe_32k.json.gz new file mode 100644 index 00000000..e9d0aa23 Binary files /dev/null and b/marker_experiments/tokenizers/ru_bnd_w_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ru_bnd_wp_bpe_32k.json.gz b/marker_experiments/tokenizers/ru_bnd_wp_bpe_32k.json.gz new file mode 100644 index 00000000..ab217a76 Binary files /dev/null and b/marker_experiments/tokenizers/ru_bnd_wp_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ru_bnd_wpd_bpe_32k.json.gz b/marker_experiments/tokenizers/ru_bnd_wpd_bpe_32k.json.gz new file mode 100644 index 00000000..f4d635b2 Binary files /dev/null and b/marker_experiments/tokenizers/ru_bnd_wpd_bpe_32k.json.gz differ diff --git a/marker_experiments/tokenizers/ru_plain_bpe_32k.json.gz b/marker_experiments/tokenizers/ru_plain_bpe_32k.json.gz new file mode 100644 index 00000000..f9ab813e Binary files /dev/null and b/marker_experiments/tokenizers/ru_plain_bpe_32k.json.gz differ