diff --git a/marker_experiments/downstream/cluster/submit_extcaps.sh b/marker_experiments/downstream/cluster/submit_extcaps.sh new file mode 100755 index 00000000..f0c12ce3 --- /dev/null +++ b/marker_experiments/downstream/cluster/submit_extcaps.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# The bnd_wpd_extcaps downstream sweep: one arm, seeds 0-2, matched to the BPE runs. +# +# CORE_SAFE_ARMS="" marker_experiments/downstream/cluster/submit_extcaps.sh +# +# bnd_wpd_extcaps differs from bnd_wpd_caps only in which side of the boundary markers the +# caps code sits on: `The` is `<^><|>the<|>` here against `<|><^>the<|>` there. The +# lowercase span is therefore a suffix of the cased one and BPE may cover both with one +# piece. Every other setting matches the committed BPE runs so the difference in the +# downstream numbers is attributable to that one change. +# +# ONE job for all three seeds, not three. A node carries 4 GPUs and a node-hour bills as 4 +# GPU-hours whether or not a GPU is touched, so three single-GPU runs on three nodes bill +# 12 GPU-hours an hour to use 3. They run concurrently on GPUs 0, 1 and 2 of one node. +# Concurrency is safe because runner.py gives each run its own base dir keyed by +# tokenizer_id, which carries the seed, and symlinks the shared read-only ClimbMix shards +# and CORE bundle into it; the bpb byte table that concurrent runs used to race on is +# per-run. ENCODE_WORKERS is set because run_downstream_eval otherwise sizes its encode +# pool for a run that owns the whole node. + +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "${REPO}" +source marker_experiments/downstream/cluster/env.sh + +ARM=${ARM:-bnd_wpd_extcaps} +# Which arms CORE can score. Required, and MAY be empty: run core_prefix_check.py on the +# trained tokenizer and pass what it supports. Getting it wrong loses the whole run: +# base_eval raises on the first violation and reports nothing, bits-per-byte included. +# No `:` in the test, so an explicit empty value is honoured. +: "${CORE_SAFE_ARMS?set CORE_SAFE_ARMS explicitly; the empty string is a valid value}" +# No default. a0229 and infra01 bill different budgets and carry different fairshare. +: "${ACCOUNT:?set ACCOUNT (a0229 or infra01)}" +SEEDS=${SEEDS:-0,1,2} +TRAINER=${TRAINER:-bpe} +DEPTH=${DEPTH:-12} +VOCAB=${VOCAB:-34685} +CORPUS=${CORPUS:-fineweb_en_5gb} +NUM_SHARDS=${NUM_SHARDS:-8} +GPUS=${GPUS:-1} +TRAIN_WORKERS=${TRAIN_WORKERS:-90} +ENCODE_WORKERS=${ENCODE_WORKERS:-90} +# Not `${OUT:-...}`: env.sh has already exported OUT=results/marker_downstream, so +# defaulting from OUT would drop this sweep's logs in with the committed BPE runs and +# collect_results.py would then rebuild one TSV over both. +OUT_DIR=${OUT_DIR:-results/extcaps_downstream} +NANOCHAT_BASE_DIR=/capstor/scratch/cscs/cmeister747/marker_downstream/nanochat_base + +# `git status --porcelain`, not `git diff`. Neither `git diff HEAD` nor +# `git diff --cached` reports untracked files, so the guard passed while a new, +# untracked file that the job runs was described by no commit. +DIRT=$(git status --porcelain) +if [[ -n "${DIRT}" ]]; then + echo "refusing to submit: working tree differs from HEAD, so the jobs would run code" >&2 + echo "no commit describes. Untracked files count." >&2 + echo "${DIRT}" >&2 + exit 1 +fi + +TOK="marker_experiments/downstream/tokenizers/${CORPUS}_${ARM}_${TRAINER}_v${VOCAB}.json.gz" +[[ -f "$TOK" ]] || { echo "missing tokenizer: $TOK" >&2; exit 1; } + +# Verify the tokenizer is exactly the matched size before spending any GPU time. The +# adapter reports vocab+1 for the synthetic BOS. +n=$(PYTHONPATH="${REPO}:${REPO}/eval/py-nanochat" uv run python -c " +from marker_experiments.downstream.boundary_tokenizer import BoundaryBPETokenizer +from pynanochat.tokenizer_adapter import ScriptBPETokenizerAdapter +print(ScriptBPETokenizerAdapter(BoundaryBPETokenizer.load('$TOK')).get_vocab_size()) +") +if [[ "$n" != "$((VOCAB + 1))" ]]; then + echo "refusing to submit: ${ARM} vocabulary is ${n}, expected $((VOCAB + 1))" >&2 + exit 1 +fi +echo "-- ${ARM}: vocabulary ${n}, matched" + +# The shards must already be there. runner.py refuses to download from inside a run, so a +# missing shard would fail three runs an hour in rather than here. +have=$(ls "${NANOCHAT_BASE_DIR}/base_data_climbmix"/shard_*.parquet 2>/dev/null | wc -l) +if (( have < NUM_SHARDS + 1 )); then + echo "refusing to submit: ${NANOCHAT_BASE_DIR}/base_data_climbmix holds ${have} shard(s)," >&2 + echo "and this sweep declares ${NUM_SHARDS} train shards plus a val shard." >&2 + exit 1 +fi +echo "-- shards: ${have} present, ${NUM_SHARDS} train + 1 val required" + +mkdir -p "${OUT_DIR}/slurm" "${OUT_DIR}/logs" +# No `|| true`. A scheduler hiccup would leave IN_FLIGHT empty, every seed would look +# absent, and a re-run would submit duplicates. Two jobs with one tokenizer_id share +# NANOCHAT_BASE/runs/, hence the checkpoint directory and token_bytes.pt, and both +# tee to one log. That is how the first sweep of this project was lost. +if ! IN_FLIGHT=$(squeue -u "$USER" -h -o "%j" 2>&1); then + echo "refusing to submit: squeue failed, so in-flight jobs cannot be detected and" >&2 + echo "this would submit duplicates sharing a checkpoint directory and a log." >&2 + echo "${IN_FLIGHT}" >&2 + exit 1 +fi +JOB="extcaps_${ARM}_${TRAINER}_d${DEPTH}" +if grep -qx "${JOB}" <<< "$IN_FLIGHT"; then + echo "-- ${JOB}: already queued or running, not submitting"; exit 0 +fi + +IFS=',' read -ra SEED_LIST <<< "$SEEDS" +TODO=() +for seed in "${SEED_LIST[@]}"; do + log="${OUT_DIR}/logs/${ARM}_${TRAINER}_d${DEPTH}_s${seed}.log" + if [[ -s "$log" ]] && grep -q "artifact_dir" "$log"; then + echo "-- seed ${seed}: already has a result, not submitting"; continue + fi + TODO+=("$seed") +done +if (( ${#TODO[@]} == 0 )); then echo "nothing to do"; exit 0; fi +# One GPU per seed, one node, so the node's 4 GPUs are the ceiling. Past that the loop +# below would hand two seeds the same CUDA_VISIBLE_DEVICES and they would share a GPU. +GPUS_PER_NODE=4 +if (( ${#TODO[@]} > GPUS_PER_NODE )); then + echo "refusing to submit: ${#TODO[@]} seeds but only ${GPUS_PER_NODE} GPUs on a node." >&2 + echo "Run them in batches, or extend this script to allocate more nodes." >&2 + exit 1 +fi +echo "-- submitting seeds: ${TODO[*]}" + +# Build the concurrent launch block: one run_arms.sh per seed, each pinned to its own GPU. +LAUNCH="" +gpu=0 +for seed in "${TODO[@]}"; do + LAUNCH+=" +CUDA_VISIBLE_DEVICES=${gpu} ARMS=${ARM} SEEDS=${seed} \\ + marker_experiments/downstream/run_arms.sh > \"${OUT_DIR}/slurm/\${SLURM_JOB_ID}_s${seed}.out\" 2>&1 & +pids+=(\$!); seeds+=(${seed}) +echo \"[pack] seed ${seed} on GPU ${gpu}, pid \${pids[-1]}\" +" + gpu=$((gpu + 1)) +done + +jid=$(sbatch --parsable --job-name="${JOB}" --account="${ACCOUNT}" --partition=normal \ + --nodes=1 --ntasks-per-node=1 --time=12:00:00 \ + --output="${OUT_DIR}/slurm/%x_%j.out" --error="${OUT_DIR}/slurm/%x_%j.err" \ + --wrap=" +# No -e: a failing seed must reach the wait loop below and be reported, not abort the job. +set -uo pipefail +cd ${REPO} || exit 1 +source marker_experiments/downstream/cluster/env.sh +# The venv installs script_bpe and pynanochat as editable pointers into the MAIN +# checkout, so without this the job runs that tree's library code no matter which +# checkout the driver script came from, and the clean-tree guard checks the wrong +# tree. Both entries are needed: REPO alone still resolves pynanochat to the main one. +export PYTHONPATH=\"${REPO}:${REPO}/eval/py-nanochat:\${PYTHONPATH:-}\" +export NANOCHAT_BASE=${NANOCHAT_BASE_DIR} +export OUT=${OUT_DIR} +export TRAINER=${TRAINER} DEPTH=${DEPTH} GPUS=${GPUS} +export CORE_SAFE_ARMS=\"${CORE_SAFE_ARMS}\" +# run_arms.sh ends by copying \$OUT/results.tsv over marker_experiments/paper/generated +# and regenerating the tables. This sweep TSV holds three extcaps rows, and those paper +# artifacts are tracked and carry the full BPE grid. Keep them out of it. +export SKIP_PAPER_ARTIFACTS=1 +export VOCAB=${VOCAB} CORPUS=${CORPUS} NUM_SHARDS=${NUM_SHARDS} +export TRAIN_WORKERS=${TRAIN_WORKERS} ENCODE_WORKERS=${ENCODE_WORKERS} +# Pinned, not inherited. --wrap runs under the submitting shell environment, so a +# leftover SMOKE=1 would make every seed train 20 iterations, tag itself _smoke, and +# still print a result block that collect_results.py parses into a row. +export SMOKE=0 +export TAG_SUFFIX= + +COMMIT=\$(git rev-parse HEAD) +MAIN_COMMIT=\$(git -C /users/cmeister747/script_tok rev-parse HEAD) +MAIN_DIRTY=\$(git -C /users/cmeister747/script_tok status --porcelain | wc -l) +VENDOR=\$(git -C eval/py-nanochat/vendor/nanochat rev-parse HEAD) +TOK_SHA=\$(sha256sum ${TOK} | cut -d' ' -f1) +echo \"[job] \${SLURM_JOB_ID} ${JOB} commit=\${COMMIT}\" +echo \"[job] main_checkout=\${MAIN_COMMIT} dirty=\${MAIN_DIRTY} nanochat=\${VENDOR}\" +echo \"[job] tokenizer=${TOK}\" +echo \"[job] tokenizer_sha256=\${TOK_SHA}\" +echo \"[job] node=\$(hostname) cores=\$(nproc) gpus=\$(nvidia-smi -L | wc -l)\" +echo \"[job] seeds=${TODO[*]} core_safe_arms='${CORE_SAFE_ARMS}' encode_workers=${ENCODE_WORKERS}\" +uv pip freeze > \"${OUT_DIR}/slurm/\${SLURM_JOB_ID}.freeze\" 2>/dev/null || true + +# Count the GPUs on the node that was actually allocated. The submit-side check used a +# constant, which is the login node's answer to a question only the compute node can +# settle; two runs handed the same device would not be clean single-GPU replicates. +NGPU=\$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l) +if (( ${#TODO[@]} > NGPU )); then + echo \"[pack] refusing: ${#TODO[@]} seeds on a node with \${NGPU} GPU(s)\" >&2 + exit 1 +fi +echo \"[job] allocated GPUs: \${NGPU}\" + +pids=(); seeds=() +${LAUNCH} +# Report every failure rather than dying on the first, so one bad seed does not hide the +# other two. +fail=0 +for i in \"\${!pids[@]}\"; do + if wait \"\${pids[\$i]}\"; then + echo \"[pack] seed \${seeds[\$i]} ok\" + else + echo \"[pack] seed \${seeds[\$i]} FAILED\" + fail=1 + fi +done + +# One authoritative collection after the join. Each run_arms.sh also collects when it +# finishes, and three concurrent writers to one TSV can interleave; this pass runs when +# every log is complete and overwrites whatever they left. +uv run python marker_experiments/downstream/collect_results.py \\ + --logs-dir \"${OUT_DIR}/logs\" --out \"${OUT_DIR}/results.tsv\" || fail=1 +echo \"[pack] done, fail=\${fail}\" +exit \${fail} +") +echo "-- ${JOB}: job ${jid}, ${#TODO[@]} seed(s) on one node" diff --git a/marker_experiments/downstream/extcaps_preflight.py b/marker_experiments/downstream/extcaps_preflight.py new file mode 100644 index 00000000..796cfe9a --- /dev/null +++ b/marker_experiments/downstream/extcaps_preflight.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Tokenizer-side checks for the bnd_wpd_extcaps downstream sweep, before any GPU time. + +Three sweeps have been discarded for measurement bugs, so every property the sweep relies +on is measured here rather than assumed: + + * vocabulary is exactly the matched size (the adapter reports vocab+1, for the synthetic + BOS it adds on top of the trained vocabulary); + * the held-out slice round-trips with zero failures; + * chars/token beside the arms this one is meant to be compared against; + * only BOS carries zero bytes in the table evaluate_bpb masks on, and every token that + decodes to the empty string carries the floored count of 1. A boundary marker decodes + to nothing, and without the floor the cost of predicting it would be dropped from the + bpb numerator for exactly the arms under test. + +The CORE prefix property is measured separately by core_prefix_check.py, which needs the +CORE bundle. + + uv run python marker_experiments/downstream/extcaps_preflight.py +""" + +import json +import os +import sys + +import cyclopts + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, REPO) + +from marker_experiments.downstream.train_matched import eval_compression # noqa: E402 + +app = cyclopts.App() + + +@app.default +def main( + arm: str = "bnd_wpd_extcaps", + against: str = "bnd_wpd_caps,plain", + corpus: str = "fineweb_en_5gb", + trainer: str = "bpe", + vocab: int = 34_685, + eval_texts: str = os.path.join(REPO, "marker_experiments", "eval_texts", "en.json"), +) -> None: + """Check the trained tokenizer and print the numbers the sweep is gated on. + + Args: + arm: The arm under test. + against: Comma-separated arms to report chars/token beside it. + corpus: Corpus name in the tokenizer filename. + trainer: Trainer name in the tokenizer filename. + vocab: Matched total vocabulary the arm must land on exactly. + eval_texts: Held-out slice for chars/token and round-trip. + """ + from marker_experiments.downstream.boundary_tokenizer import BoundaryBPETokenizer + from pynanochat.tokenizer_adapter import ScriptBPETokenizerAdapter + from pynanochat.tokenizer import special_aware_token_bytes, special_token_ids + + tok_dir = os.path.join(HERE, "tokenizers") + path = os.path.join(tok_dir, f"{corpus}_{arm}_{trainer}_v{vocab}.json.gz") + if not os.path.exists(path): + raise SystemExit(f"missing tokenizer: {path}") + + failures = [] + print(f"tokenizer: {path}") + tokenizer = BoundaryBPETokenizer.load(path) + adapter = ScriptBPETokenizerAdapter(tokenizer) + + # 1. vocabulary + n_adapter = adapter.get_vocab_size() + n_trained = len(tokenizer.tokens) + print(f"\n[1] vocabulary trained={n_trained:,} adapter={n_adapter:,} (+1 synthetic BOS)") + if n_trained != vocab: + failures.append(f"trained vocabulary {n_trained} != {vocab}") + if n_adapter != vocab + 1: + failures.append(f"adapter vocabulary {n_adapter} != {vocab + 1}") + + # 2 + 3. round-trip and chars/token, beside the comparison arms + print(f"\n[2/3] held-out slice {os.path.relpath(eval_texts, REPO)}") + stats = eval_compression(tokenizer, eval_texts) + if not stats: + raise SystemExit(f"no eval slice at {eval_texts}") + print(f" {arm:<20} chars/token {stats['eval_chars_per_token']:.4f} " + f"roundtrip_failures {stats['roundtrip_failures']} " + f"({stats['eval_chars']:,} chars, {stats['eval_tokens']:,} tokens)") + if stats["roundtrip_failures"] != 0: + failures.append(f"{stats['roundtrip_failures']} round-trip failures on the held-out slice") + + base = stats["eval_chars_per_token"] + for other in [a.strip() for a in against.split(",") if a.strip()]: + other_path = os.path.join(tok_dir, f"{corpus}_{other}_{trainer}_v{vocab}.json.gz") + if not os.path.exists(other_path): + print(f" {other:<20} MISSING at {other_path}") + failures.append(f"comparison tokenizer missing: {other_path}") + continue + other_tok = BoundaryBPETokenizer.load(other_path) + s = eval_compression(other_tok, eval_texts) + delta = 100 * (base - s["eval_chars_per_token"]) / s["eval_chars_per_token"] + print(f" {other:<20} chars/token {s['eval_chars_per_token']:.4f} " + f"roundtrip_failures {s['roundtrip_failures']} " + f"({arm} is {delta:+.2f}% against it)") + + # 4. the byte table evaluate_bpb masks on + print("\n[4] byte table (0 bytes -> loss masked by evaluate_bpb)") + counts = special_aware_token_bytes(adapter) + specials = special_token_ids(adapter) + bos = adapter.get_bos_token_id() + zeros = [i for i, c in enumerate(counts) if c == 0] + empty = [i for i in range(n_adapter) if i not in specials and adapter.decode([i]) == ""] + print(f" special ids {sorted(specials)} bos id {bos}") + print(f" ids with 0 bytes: {zeros}") + print(f" non-special ids decoding to the empty string: {len(empty)}") + if empty: + bad = [i for i in empty if counts[i] != 1] + print(f" of those, byte count 1: {len(empty) - len(bad)}; not 1: {len(bad)}") + if bad: + failures.append(f"{len(bad)} empty-decoding tokens are not floored to 1 byte") + if zeros != [bos]: + failures.append(f"ids with 0 bytes are {zeros}, expected exactly [{bos}] (BOS)") + + print("\n" + "=" * 60) + if failures: + for f in failures: + print(f"FAIL: {f}") + raise SystemExit(1) + print("all tokenizer-side checks passed") + + +if __name__ == "__main__": + app() diff --git a/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_w_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_w_mingram_v34685.json.gz new file mode 100644 index 00000000..1e5b117e Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_w_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_wp_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_wp_mingram_v34685.json.gz new file mode 100644 index 00000000..314e26c8 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_wp_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_wpd_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_wpd_mingram_v34685.json.gz new file mode 100644 index 00000000..b51e5577 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_bnd_wpd_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_plain_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_plain_mingram_v34685.json.gz new file mode 100644 index 00000000..cd41c1ca Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ar_5gb_plain_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_w_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_w_mingram_v34685.json.gz new file mode 100644 index 00000000..21555152 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_w_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_wp_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_wp_mingram_v34685.json.gz new file mode 100644 index 00000000..b1adb1aa Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_wp_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_wpd_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_wpd_mingram_v34685.json.gz new file mode 100644 index 00000000..2bbb18da Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_bnd_wpd_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_de_5gb_plain_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_plain_mingram_v34685.json.gz new file mode 100644 index 00000000..1f50ae3a Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_de_5gb_plain_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_w_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_w_mingram_v34685.json.gz new file mode 100644 index 00000000..919d12aa Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_w_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wp_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wp_mingram_v34685.json.gz new file mode 100644 index 00000000..4837b8c0 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wp_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685.json.gz new file mode 100644 index 00000000..e5c94121 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_w_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_w_mingram_v34685.json.gz new file mode 100644 index 00000000..dc011345 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_w_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_wp_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_wp_mingram_v34685.json.gz new file mode 100644 index 00000000..6d833a57 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_wp_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_wpd_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_wpd_mingram_v34685.json.gz new file mode 100644 index 00000000..c4659bd2 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_bnd_wpd_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_plain_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_plain_mingram_v34685.json.gz new file mode 100644 index 00000000..a79899fc Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_fi_5gb_plain_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_w_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_w_mingram_v34685.json.gz new file mode 100644 index 00000000..0b46059c Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_w_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_wp_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_wp_mingram_v34685.json.gz new file mode 100644 index 00000000..2f8e8573 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_wp_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_wpd_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_wpd_mingram_v34685.json.gz new file mode 100644 index 00000000..f2fe9239 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_bnd_wpd_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_plain_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_plain_mingram_v34685.json.gz new file mode 100644 index 00000000..7a51ac1e Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_ru_5gb_plain_mingram_v34685.json.gz differ diff --git a/marker_experiments/paper/generated/eval_goldfish.json b/marker_experiments/paper/generated/eval_goldfish.json index 10b099e4..1981131f 100644 --- a/marker_experiments/paper/generated/eval_goldfish.json +++ b/marker_experiments/paper/generated/eval_goldfish.json @@ -11,6 +11,18 @@ "train_corpus": "fineweb_ar_5gb", "train_tokens": 1329072646 }, + "fineweb_ar_5gb_bnd_w_mingram_v34685": { + "eval_chars": 8036366, + "eval_chars_per_token": 3.658174710400105, + "eval_corpus": "goldfish:arb_arab.txt", + "eval_docs": 31081, + "eval_tokens": 2196824, + "roundtrip_failures": 0, + "slice_hash": "aec9fd6c5acc275c", + "train_chars": 4997549743, + "train_corpus": "fineweb_ar_5gb", + "train_tokens": 1321433705 + }, "fineweb_ar_5gb_bnd_wp_bpe_v34685": { "eval_chars": 8036366, "eval_chars_per_token": 3.8977919015372198, @@ -23,6 +35,18 @@ "train_corpus": "fineweb_ar_5gb", "train_tokens": 1237944737 }, + "fineweb_ar_5gb_bnd_wp_mingram_v34685": { + "eval_chars": 8036366, + "eval_chars_per_token": 3.9236006819606564, + "eval_corpus": "goldfish:arb_arab.txt", + "eval_docs": 31081, + "eval_tokens": 2048212, + "roundtrip_failures": 0, + "slice_hash": "aec9fd6c5acc275c", + "train_chars": 4997549743, + "train_corpus": "fineweb_ar_5gb", + "train_tokens": 1230294575 + }, "fineweb_ar_5gb_bnd_wpd_bpe_v34685": { "eval_chars": 8036366, "eval_chars_per_token": 3.9653428241961737, @@ -47,6 +71,18 @@ "train_corpus": "fineweb_ar_5gb", "train_tokens": 1253230657 }, + "fineweb_ar_5gb_bnd_wpd_mingram_v34685": { + "eval_chars": 8036366, + "eval_chars_per_token": 3.9922334823646297, + "eval_corpus": "goldfish:arb_arab.txt", + "eval_docs": 31081, + "eval_tokens": 2013000, + "roundtrip_failures": 0, + "slice_hash": "aec9fd6c5acc275c", + "train_chars": 4997549743, + "train_corpus": "fineweb_ar_5gb", + "train_tokens": 1210371332 + }, "fineweb_ar_5gb_plain_bpe_v34685": { "eval_chars": 8036366, "eval_chars_per_token": 3.973489193550939, @@ -59,6 +95,18 @@ "train_corpus": "fineweb_ar_5gb", "train_tokens": 1222664188 }, + "fineweb_ar_5gb_plain_mingram_v34685": { + "eval_chars": 8036366, + "eval_chars_per_token": 4.023890889226368, + "eval_corpus": "goldfish:arb_arab.txt", + "eval_docs": 31081, + "eval_tokens": 1997163, + "roundtrip_failures": 0, + "slice_hash": "aec9fd6c5acc275c", + "train_chars": 4997549743, + "train_corpus": "fineweb_ar_5gb", + "train_tokens": 1208558599 + }, "fineweb_ar_5gb_quick_bnd_w_bpe_v34685": { "eval_chars": 8036366, "eval_chars_per_token": 3.63050661240731, @@ -263,6 +311,18 @@ "train_corpus": "fineweb_de_5gb", "train_tokens": 1216560299 }, + "fineweb_de_5gb_bnd_w_mingram_v34685": { + "eval_chars": 8033174, + "eval_chars_per_token": 4.153148760984306, + "eval_corpus": "goldfish:deu_latn.txt", + "eval_docs": 23953, + "eval_tokens": 1934237, + "roundtrip_failures": 0, + "slice_hash": "eca22877fba9c9eb", + "train_chars": 4997457262, + "train_corpus": "fineweb_de_5gb", + "train_tokens": 1196858906 + }, "fineweb_de_5gb_bnd_wp_bpe_v34685": { "eval_chars": 8033174, "eval_chars_per_token": 4.4384604461791, @@ -275,6 +335,18 @@ "train_corpus": "fineweb_de_5gb", "train_tokens": 1125504486 }, + "fineweb_de_5gb_bnd_wp_mingram_v34685": { + "eval_chars": 8033174, + "eval_chars_per_token": 4.518179471272527, + "eval_corpus": "goldfish:deu_latn.txt", + "eval_docs": 23953, + "eval_tokens": 1777967, + "roundtrip_failures": 0, + "slice_hash": "eca22877fba9c9eb", + "train_chars": 4997457262, + "train_corpus": "fineweb_de_5gb", + "train_tokens": 1105770329 + }, "fineweb_de_5gb_bnd_wpd_bpe_v34685": { "eval_chars": 8033174, "eval_chars_per_token": 4.527109092343586, @@ -299,6 +371,18 @@ "train_corpus": "fineweb_de_5gb", "train_tokens": 1102478035 }, + "fineweb_de_5gb_bnd_wpd_mingram_v34685": { + "eval_chars": 8033174, + "eval_chars_per_token": 4.610899694297027, + "eval_corpus": "goldfish:deu_latn.txt", + "eval_docs": 23953, + "eval_tokens": 1742214, + "roundtrip_failures": 0, + "slice_hash": "eca22877fba9c9eb", + "train_chars": 4997457262, + "train_corpus": "fineweb_de_5gb", + "train_tokens": 1085260981 + }, "fineweb_de_5gb_plain_bpe_v34685": { "eval_chars": 8033174, "eval_chars_per_token": 4.574380040578023, @@ -311,6 +395,18 @@ "train_corpus": "fineweb_de_5gb", "train_tokens": 1098186861 }, + "fineweb_de_5gb_plain_mingram_v34685": { + "eval_chars": 8033174, + "eval_chars_per_token": 4.665331697143889, + "eval_corpus": "goldfish:deu_latn.txt", + "eval_docs": 23953, + "eval_tokens": 1721887, + "roundtrip_failures": 0, + "slice_hash": "eca22877fba9c9eb", + "train_chars": 4997457262, + "train_corpus": "fineweb_de_5gb", + "train_tokens": 1076721751 + }, "fineweb_de_5gb_quick_bnd_w_bpe_v34685": { "eval_chars": 8033174, "eval_chars_per_token": 4.084043140552506, @@ -515,6 +611,18 @@ "train_corpus": "fineweb_en_5gb", "train_tokens": 1255889200 }, + "fineweb_en_5gb_bnd_w_mingram_v34685": { + "eval_chars": 8109368, + "eval_chars_per_token": 4.086721855990034, + "eval_corpus": "goldfish:eng_latn.txt", + "eval_docs": 22442, + "eval_tokens": 1984321, + "roundtrip_failures": 0, + "slice_hash": "8bca50f02b720acc", + "train_chars": 4994221434, + "train_corpus": "fineweb_en_5gb", + "train_tokens": 1248287691 + }, "fineweb_en_5gb_bnd_wp_bpe_v34685": { "eval_chars": 8109368, "eval_chars_per_token": 4.399205585853309, @@ -527,6 +635,18 @@ "train_corpus": "fineweb_en_5gb", "train_tokens": 1162736882 }, + "fineweb_en_5gb_bnd_wp_mingram_v34685": { + "eval_chars": 8109368, + "eval_chars_per_token": 4.4284568273722895, + "eval_corpus": "goldfish:eng_latn.txt", + "eval_docs": 22442, + "eval_tokens": 1831195, + "roundtrip_failures": 0, + "slice_hash": "8bca50f02b720acc", + "train_chars": 4994221434, + "train_corpus": "fineweb_en_5gb", + "train_tokens": 1155078497 + }, "fineweb_en_5gb_bnd_wpd_bpe_v34685": { "eval_chars": 8109368, "eval_chars_per_token": 4.484629958778006, @@ -551,6 +671,18 @@ "train_corpus": "fineweb_en_5gb", "train_tokens": 1140849054 }, + "fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685": { + "eval_chars": 8109368, + "eval_chars_per_token": 4.490657209893949, + "eval_corpus": "goldfish:eng_latn.txt", + "eval_docs": 22442, + "eval_tokens": 1805831, + "roundtrip_failures": 0, + "slice_hash": "8bca50f02b720acc", + "train_chars": 4994221434, + "train_corpus": "fineweb_en_5gb", + "train_tokens": 1140257359 + }, "fineweb_en_5gb_bnd_wpd_mingram_v34685": { "eval_chars": 8109368, "eval_chars_per_token": 4.51586439322601, @@ -791,6 +923,18 @@ "train_corpus": "fineweb_fi_5gb", "train_tokens": 1161948277 }, + "fineweb_fi_5gb_bnd_w_mingram_v34685": { + "eval_chars": 8013043, + "eval_chars_per_token": 4.6646689750241, + "eval_corpus": "goldfish:fin_latn.txt", + "eval_docs": 29307, + "eval_tokens": 1717816, + "roundtrip_failures": 0, + "slice_hash": "1df047431ded0222", + "train_chars": 4994960579, + "train_corpus": "fineweb_fi_5gb", + "train_tokens": 1143138540 + }, "fineweb_fi_5gb_bnd_wp_bpe_v34685": { "eval_chars": 8013043, "eval_chars_per_token": 4.928285921463975, @@ -803,6 +947,18 @@ "train_corpus": "fineweb_fi_5gb", "train_tokens": 1090389249 }, + "fineweb_fi_5gb_bnd_wp_mingram_v34685": { + "eval_chars": 8013043, + "eval_chars_per_token": 4.974218378995788, + "eval_corpus": "goldfish:fin_latn.txt", + "eval_docs": 29307, + "eval_tokens": 1610915, + "roundtrip_failures": 0, + "slice_hash": "1df047431ded0222", + "train_chars": 4994960579, + "train_corpus": "fineweb_fi_5gb", + "train_tokens": 1071616061 + }, "fineweb_fi_5gb_bnd_wpd_bpe_v34685": { "eval_chars": 8013043, "eval_chars_per_token": 4.988341922141695, @@ -827,6 +983,18 @@ "train_corpus": "fineweb_fi_5gb", "train_tokens": 1071141065 }, + "fineweb_fi_5gb_bnd_wpd_mingram_v34685": { + "eval_chars": 8013043, + "eval_chars_per_token": 5.035618024059982, + "eval_corpus": "goldfish:fin_latn.txt", + "eval_docs": 29307, + "eval_tokens": 1591273, + "roundtrip_failures": 0, + "slice_hash": "1df047431ded0222", + "train_chars": 4994960579, + "train_corpus": "fineweb_fi_5gb", + "train_tokens": 1055162976 + }, "fineweb_fi_5gb_plain_bpe_v34685": { "eval_chars": 8013043, "eval_chars_per_token": 5.011788554069547, @@ -839,6 +1007,18 @@ "train_corpus": "fineweb_fi_5gb", "train_tokens": 1071826379 }, + "fineweb_fi_5gb_plain_mingram_v34685": { + "eval_chars": 8013043, + "eval_chars_per_token": 5.152810591338592, + "eval_corpus": "goldfish:fin_latn.txt", + "eval_docs": 29307, + "eval_tokens": 1555082, + "roundtrip_failures": 0, + "slice_hash": "1df047431ded0222", + "train_chars": 4994960579, + "train_corpus": "fineweb_fi_5gb", + "train_tokens": 1038682926 + }, "fineweb_fi_5gb_quick_bnd_w_bpe_v34685": { "eval_chars": 8013043, "eval_chars_per_token": 4.622411624901933, @@ -1355,6 +1535,18 @@ "train_corpus": "fineweb_ru_5gb", "train_tokens": 1280518773 }, + "fineweb_ru_5gb_bnd_w_mingram_v34685": { + "eval_chars": 8015461, + "eval_chars_per_token": 3.890271531645041, + "eval_corpus": "goldfish:rus_cyrl.txt", + "eval_docs": 27615, + "eval_tokens": 2060386, + "roundtrip_failures": 0, + "slice_hash": "0896372a26fffac9", + "train_chars": 4999671485, + "train_corpus": "fineweb_ru_5gb", + "train_tokens": 1268156020 + }, "fineweb_ru_5gb_bnd_wp_bpe_v34685": { "eval_chars": 8015461, "eval_chars_per_token": 4.268870090016584, @@ -1367,6 +1559,18 @@ "train_corpus": "fineweb_ru_5gb", "train_tokens": 1162777299 }, + "fineweb_ru_5gb_bnd_wp_mingram_v34685": { + "eval_chars": 8015461, + "eval_chars_per_token": 4.3162588400135915, + "eval_corpus": "goldfish:rus_cyrl.txt", + "eval_docs": 27615, + "eval_tokens": 1857039, + "roundtrip_failures": 0, + "slice_hash": "0896372a26fffac9", + "train_chars": 4999671485, + "train_corpus": "fineweb_ru_5gb", + "train_tokens": 1150370586 + }, "fineweb_ru_5gb_bnd_wpd_bpe_v34685": { "eval_chars": 8015461, "eval_chars_per_token": 4.343928875304166, @@ -1391,6 +1595,18 @@ "train_corpus": "fineweb_ru_5gb", "train_tokens": 1139746207 }, + "fineweb_ru_5gb_bnd_wpd_mingram_v34685": { + "eval_chars": 8015461, + "eval_chars_per_token": 4.394340152255266, + "eval_corpus": "goldfish:rus_cyrl.txt", + "eval_docs": 27615, + "eval_tokens": 1824042, + "roundtrip_failures": 0, + "slice_hash": "0896372a26fffac9", + "train_chars": 4999671485, + "train_corpus": "fineweb_ru_5gb", + "train_tokens": 1130568761 + }, "fineweb_ru_5gb_plain_bpe_v34685": { "eval_chars": 8015461, "eval_chars_per_token": 4.32163225914284, @@ -1403,6 +1619,18 @@ "train_corpus": "fineweb_ru_5gb", "train_tokens": 1152140966 }, + "fineweb_ru_5gb_plain_mingram_v34685": { + "eval_chars": 8015461, + "eval_chars_per_token": 4.373656876204528, + "eval_corpus": "goldfish:rus_cyrl.txt", + "eval_docs": 27615, + "eval_tokens": 1832668, + "roundtrip_failures": 0, + "slice_hash": "0896372a26fffac9", + "train_chars": 4999671485, + "train_corpus": "fineweb_ru_5gb", + "train_tokens": 1139033861 + }, "fineweb_ru_5gb_quick_bnd_w_bpe_v34685": { "eval_chars": 8015461, "eval_chars_per_token": 3.845729159169965, diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_w_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_w_mingram_v34685.json new file mode 100644 index 00000000..d78dd629 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_w_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ar_5gb_bnd_w_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_w", + "atomic_vocab": 1711, + "corpus": "fineweb_ar_5gb", + "eval_chars": 745245, + "eval_chars_per_token": 3.3515999190483687, + "eval_slice": "marker_experiments/eval_texts/ar.json", + "eval_tokens": 222355, + "lang": "ar", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ar_5gb_bnd_w_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 782, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_wp_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_wp_mingram_v34685.json new file mode 100644 index 00000000..4023cc44 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_wp_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ar_5gb_bnd_wp_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wp", + "atomic_vocab": 1711, + "corpus": "fineweb_ar_5gb", + "eval_chars": 745245, + "eval_chars_per_token": 3.683532854219595, + "eval_slice": "marker_experiments/eval_texts/ar.json", + "eval_tokens": 202318, + "lang": "ar", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ar_5gb_bnd_wp_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 784, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_wpd_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_wpd_mingram_v34685.json new file mode 100644 index 00000000..fec0a6a4 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_bnd_wpd_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ar_5gb_bnd_wpd_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wpd", + "atomic_vocab": 1711, + "corpus": "fineweb_ar_5gb", + "eval_chars": 745245, + "eval_chars_per_token": 3.843311484371374, + "eval_slice": "marker_experiments/eval_texts/ar.json", + "eval_tokens": 193907, + "lang": "ar", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ar_5gb_bnd_wpd_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 787, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_plain_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_plain_mingram_v34685.json new file mode 100644 index 00000000..a7211465 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ar_5gb_plain_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ar_5gb_plain_mingram_v34685": { + "additional_vocab_size": 32975, + "arm": "plain", + "atomic_vocab": 1710, + "corpus": "fineweb_ar_5gb", + "eval_chars": 745245, + "eval_chars_per_token": 3.817051746303287, + "eval_slice": "marker_experiments/eval_texts/ar.json", + "eval_tokens": 195241, + "lang": "ar", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ar_5gb_plain_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 825, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_w_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_w_mingram_v34685.json new file mode 100644 index 00000000..48a91c81 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_w_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_de_5gb_bnd_w_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_w", + "atomic_vocab": 1711, + "corpus": "fineweb_de_5gb", + "eval_chars": 1404827, + "eval_chars_per_token": 3.4056329560413188, + "eval_slice": "marker_experiments/eval_texts/de.json", + "eval_tokens": 412501, + "lang": "de", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_de_5gb_bnd_w_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1262, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_wp_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_wp_mingram_v34685.json new file mode 100644 index 00000000..49177422 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_wp_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_de_5gb_bnd_wp_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wp", + "atomic_vocab": 1711, + "corpus": "fineweb_de_5gb", + "eval_chars": 1404827, + "eval_chars_per_token": 3.725037188669189, + "eval_slice": "marker_experiments/eval_texts/de.json", + "eval_tokens": 377131, + "lang": "de", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_de_5gb_bnd_wp_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1250, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_wpd_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_wpd_mingram_v34685.json new file mode 100644 index 00000000..a69f5437 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_bnd_wpd_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_de_5gb_bnd_wpd_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wpd", + "atomic_vocab": 1711, + "corpus": "fineweb_de_5gb", + "eval_chars": 1404827, + "eval_chars_per_token": 3.90280675532639, + "eval_slice": "marker_experiments/eval_texts/de.json", + "eval_tokens": 359953, + "lang": "de", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_de_5gb_bnd_wpd_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1270, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_plain_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_plain_mingram_v34685.json new file mode 100644 index 00000000..2a58b0de --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_de_5gb_plain_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_de_5gb_plain_mingram_v34685": { + "additional_vocab_size": 32975, + "arm": "plain", + "atomic_vocab": 1710, + "corpus": "fineweb_de_5gb", + "eval_chars": 1404827, + "eval_chars_per_token": 3.8938173533194376, + "eval_slice": "marker_experiments/eval_texts/de.json", + "eval_tokens": 360784, + "lang": "de", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_de_5gb_plain_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1363, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_w_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_w_mingram_v34685.json new file mode 100644 index 00000000..c7cb64bd --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_w_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_en_5gb_bnd_w_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_w", + "atomic_vocab": 1711, + "corpus": "fineweb_en_5gb", + "eval_chars": 3602925, + "eval_chars_per_token": 3.1113530047245535, + "eval_slice": "marker_experiments/eval_texts/en.json", + "eval_tokens": 1157993, + "lang": "en", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_en_5gb_bnd_w_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 850, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wp_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wp_mingram_v34685.json new file mode 100644 index 00000000..d26aaabf --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wp_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_en_5gb_bnd_wp_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wp", + "atomic_vocab": 1711, + "corpus": "fineweb_en_5gb", + "eval_chars": 3602925, + "eval_chars_per_token": 3.5446230402342276, + "eval_slice": "marker_experiments/eval_texts/en.json", + "eval_tokens": 1016448, + "lang": "en", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_en_5gb_bnd_wp_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 855, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685.json new file mode 100644 index 00000000..bff32272 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcaps", + "atomic_vocab": 1713, + "corpus": "fineweb_en_5gb", + "eval_chars": 3602925, + "eval_chars_per_token": 3.74635027388543, + "eval_slice": "marker_experiments/eval_texts/en.json", + "eval_tokens": 961716, + "lang": "en", + "overshoot_factor": null, + "path": "downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 153, + "trainer": "bpe" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_w_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_w_mingram_v34685.json new file mode 100644 index 00000000..b8b6ec18 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_w_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_fi_5gb_bnd_w_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_w", + "atomic_vocab": 1711, + "corpus": "fineweb_fi_5gb", + "eval_chars": 983997, + "eval_chars_per_token": 3.0845914157816203, + "eval_slice": "marker_experiments/eval_texts/fi.json", + "eval_tokens": 319004, + "lang": "fi", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_fi_5gb_bnd_w_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1805, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_wp_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_wp_mingram_v34685.json new file mode 100644 index 00000000..bbd6d543 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_wp_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_fi_5gb_bnd_wp_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wp", + "atomic_vocab": 1711, + "corpus": "fineweb_fi_5gb", + "eval_chars": 983997, + "eval_chars_per_token": 3.402385134574424, + "eval_slice": "marker_experiments/eval_texts/fi.json", + "eval_tokens": 289208, + "lang": "fi", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_fi_5gb_bnd_wp_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1801, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_wpd_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_wpd_mingram_v34685.json new file mode 100644 index 00000000..e6972419 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_bnd_wpd_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_fi_5gb_bnd_wpd_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wpd", + "atomic_vocab": 1711, + "corpus": "fineweb_fi_5gb", + "eval_chars": 983997, + "eval_chars_per_token": 3.584453478460429, + "eval_slice": "marker_experiments/eval_texts/fi.json", + "eval_tokens": 274518, + "lang": "fi", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_fi_5gb_bnd_wpd_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1795, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_plain_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_plain_mingram_v34685.json new file mode 100644 index 00000000..b771e761 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_fi_5gb_plain_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_fi_5gb_plain_mingram_v34685": { + "additional_vocab_size": 32975, + "arm": "plain", + "atomic_vocab": 1710, + "corpus": "fineweb_fi_5gb", + "eval_chars": 983997, + "eval_chars_per_token": 3.622163734079364, + "eval_slice": "marker_experiments/eval_texts/fi.json", + "eval_tokens": 271660, + "lang": "fi", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_fi_5gb_plain_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 1944, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_w_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_w_mingram_v34685.json new file mode 100644 index 00000000..a429595f --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_w_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ru_5gb_bnd_w_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_w", + "atomic_vocab": 1711, + "corpus": "fineweb_ru_5gb", + "eval_chars": 1894911, + "eval_chars_per_token": 3.00272397094431, + "eval_slice": "marker_experiments/eval_texts/ru.json", + "eval_tokens": 631064, + "lang": "ru", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ru_5gb_bnd_w_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 771, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_wp_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_wp_mingram_v34685.json new file mode 100644 index 00000000..c8c74ec9 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_wp_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ru_5gb_bnd_wp_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wp", + "atomic_vocab": 1711, + "corpus": "fineweb_ru_5gb", + "eval_chars": 1894911, + "eval_chars_per_token": 3.345446360001059, + "eval_slice": "marker_experiments/eval_texts/ru.json", + "eval_tokens": 566415, + "lang": "ru", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ru_5gb_bnd_wp_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 772, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_wpd_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_wpd_mingram_v34685.json new file mode 100644 index 00000000..02c0f4a0 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_bnd_wpd_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ru_5gb_bnd_wpd_mingram_v34685": { + "additional_vocab_size": 32974, + "arm": "bnd_wpd", + "atomic_vocab": 1711, + "corpus": "fineweb_ru_5gb", + "eval_chars": 1894911, + "eval_chars_per_token": 3.5068864048562016, + "eval_slice": "marker_experiments/eval_texts/ru.json", + "eval_tokens": 540340, + "lang": "ru", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ru_5gb_bnd_wpd_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 780, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_plain_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_plain_mingram_v34685.json new file mode 100644 index 00000000..8a9bc7a5 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_ru_5gb_plain_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_ru_5gb_plain_mingram_v34685": { + "additional_vocab_size": 32975, + "arm": "plain", + "atomic_vocab": 1710, + "corpus": "fineweb_ru_5gb", + "eval_chars": 1894911, + "eval_chars_per_token": 3.4422135108230454, + "eval_slice": "marker_experiments/eval_texts/ru.json", + "eval_tokens": 550492, + "lang": "ru", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ru_5gb_plain_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34684, + "train_seconds": 836, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/results_extcaps.tsv b/marker_experiments/paper/generated/results_extcaps.tsv new file mode 100644 index 00000000..febccd8d --- /dev/null +++ b/marker_experiments/paper/generated/results_extcaps.tsv @@ -0,0 +1,4 @@ +method arm variant trainer seed val_bpb_true train_bpb_true val_bpb train_bpb byte_factor depth vocab_size tokenizer_id log +bnd_wpd_extcaps_bpe bnd_wpd_extcaps bpe 0 0.879587190979435 0.7576745998851696 1.04396 0.899265 0.8425487480166243 12 34686 bnd_wpd_extcaps_bpe_d12_s0 bnd_wpd_extcaps_bpe_d12_s0.log +bnd_wpd_extcaps_bpe bnd_wpd_extcaps bpe 1 0.8787623357551267 0.7547753896432444 1.042981 0.895824 0.8425487480166243 12 34686 bnd_wpd_extcaps_bpe_d12_s1 bnd_wpd_extcaps_bpe_d12_s1.log +bnd_wpd_extcaps_bpe bnd_wpd_extcaps bpe 2 0.8795197870795937 0.7582289969613646 1.04388 0.899923 0.8425487480166243 12 34686 bnd_wpd_extcaps_bpe_d12_s2 bnd_wpd_extcaps_bpe_d12_s2.log diff --git a/marker_experiments/paper/generated/table_intrinsic.tex b/marker_experiments/paper/generated/table_intrinsic.tex index ed2f231f..288ec25a 100644 --- a/marker_experiments/paper/generated/table_intrinsic.tex +++ b/marker_experiments/paper/generated/table_intrinsic.tex @@ -11,17 +11,19 @@ & train & eval & train & eval & train & eval & train & eval & train & eval & train & eval & train & eval \\ \midrule \multicolumn{15}{l}{\emph{BPE}} \\ -\plainscheme & -- & 4.457 & -- & \textbf{4.574} & -- & \textbf{5.012} & -- & 4.322 & -- & \textbf{3.973} & \textbf{2.288} & \textbf{2.299} & -- & \textbf{4.106} \\ +\plainscheme & -- & 4.457 & -- & \textbf{4.574} & -- & \textbf{5.012} & -- & 4.322 & -- & \textbf{3.973} & \textbf{2.288} & \textbf{2.299} & -- & \underline{4.106} \\ \bnds{w} & $-8.29$ & $-8.87$ & $-9.73$ & $-10.68$ & $-7.76$ & $-7.74$ & $-10.03$ & $-10.87$ & $-8.01$ & $-8.49$ & $-8.15$ & $-9.12$ & $-8.66$ & $-9.30$ \\ \bnds{wp} & $-0.94$ & $-1.30$ & $-2.43$ & $-2.97$ & $-1.70$ & $-1.67$ & $-0.91$ & $-1.22$ & $\underline{-1.23}$ & $-1.91$ & $-3.39$ & $-4.00$ & $-1.77$ & $-2.18$ \\ -\bnds{wpd} & $\underline{+0.80}$ & $\underline{+0.62}$ & $\underline{-0.63}$ & $-1.03$ & $\underline{-0.21}$ & $-0.47$ & $\underline{+0.79}$ & $\underline{+0.52}$ & $\mathbf{+0.37}$ & $\underline{-0.21}$ & $\underline{-1.63}$ & $\underline{-2.18}$ & $\mathbf{-0.08}$ & $\underline{-0.46}$ \\ -\bnds{wpdcapsin} & $\mathbf{+0.96}$ & $\mathbf{+0.72}$ & $\mathbf{-0.39}$ & $\underline{-0.78}$ & $\mathbf{+0.06}$ & $\underline{-0.35}$ & $\mathbf{+1.09}$ & $\mathbf{+0.92}$ & $-2.44$ & $-2.52$ & $\underline{-1.63}$ & $-2.22$ & $\underline{-0.39}$ & $-0.71$ \\ +\bnds{wpd} & $+0.80$ & $+0.62$ & $\underline{-0.63}$ & $-1.03$ & $\underline{-0.21}$ & $-0.47$ & $\underline{+0.79}$ & $\underline{+0.52}$ & $\mathbf{+0.37}$ & $\underline{-0.21}$ & $\underline{-1.63}$ & $\underline{-2.18}$ & $\underline{-0.08}$ & $-0.46$ \\ +\bnds{wpdcaps} & $\mathbf{+1.01}$ & $\mathbf{+0.75}$ & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & $\mathbf{+1.01}$ & $\mathbf{+0.75}$ \\ +\bnds{wpdcapsin} & $\underline{+0.96}$ & $\underline{+0.72}$ & $\mathbf{-0.39}$ & $\underline{-0.78}$ & $\mathbf{+0.06}$ & $\underline{-0.35}$ & $\mathbf{+1.09}$ & $\mathbf{+0.92}$ & $-2.44$ & $-2.52$ & $\underline{-1.63}$ & $-2.22$ & $-0.39$ & $-0.71$ \\ \midrule \multicolumn{15}{l}{\emph{MinGram}} \\ -\plainscheme & -- & \underline{4.500} & -- & -- & -- & -- & -- & -- & -- & -- & \textbf{2.300} & \textbf{2.310} & -- & \textbf{3.405} \\ -\bnds{w} & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & $-8.36$ & $-9.33$ & $-8.36$ & $-9.33$ \\ -\bnds{wp} & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & $-3.59$ & $-4.17$ & $-3.59$ & $-4.17$ \\ -\bnds{wpd} & $\mathbf{+0.60}$ & $\mathbf{+0.35}$ & -- & -- & -- & -- & -- & -- & -- & -- & $-1.83$ & $\underline{-2.35}$ & $\mathbf{-0.61}$ & $\underline{-1.00}$ \\ +\plainscheme & -- & \underline{4.500} & \textbf{4.641} & \textbf{4.665} & \textbf{4.809} & \textbf{5.153} & \underline{4.389} & \underline{4.374} & \textbf{4.135} & \textbf{4.024} & \textbf{2.300} & \textbf{2.310} & -- & \textbf{4.171} \\ +\bnds{w} & $-8.55$ & $-9.19$ & $-10.04$ & $-10.98$ & $-9.14$ & $-9.47$ & $-10.18$ & $-11.05$ & $-8.54$ & $-9.09$ & $-8.36$ & $-9.33$ & $-9.13$ & $-9.85$ \\ +\bnds{wp} & $\underline{-1.17}$ & $-1.59$ & $-2.63$ & $-3.15$ & $-3.07$ & $-3.47$ & $-0.99$ & $-1.31$ & $-1.77$ & $-2.49$ & $-3.59$ & $-4.17$ & $-2.20$ & $-2.70$ \\ +\bnds{wpd} & $\mathbf{+0.60}$ & $\mathbf{+0.35}$ & $\underline{-0.79}$ & $\underline{-1.17}$ & $\underline{-1.56}$ & $\underline{-2.27}$ & $\mathbf{+0.75}$ & $\mathbf{+0.47}$ & $\underline{-0.15}$ & $\underline{-0.79}$ & $-1.83$ & $\underline{-2.35}$ & $\mathbf{-0.50}$ & $\underline{-0.96}$ \\ +\bnds{wpdcaps} & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- \\ \bnds{wpdcapsin} & -- & -- & -- & -- & -- & -- & -- & -- & -- & -- & $\underline{-1.82}$ & $-2.38$ & $\underline{-1.82}$ & $-2.38$ \\ \bottomrule \end{tabular}