diff --git a/marker_experiments/downstream/cluster/gated_mingram_seed.sbatch b/marker_experiments/downstream/cluster/gated_mingram_seed.sbatch new file mode 100644 index 00000000..69bbe4fd --- /dev/null +++ b/marker_experiments/downstream/cluster/gated_mingram_seed.sbatch @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# One seed of the MinGram downstream sweep for bnd_w, bnd_wpd_caps and bnd_wpd_extcaps, +# with the tokenizer-side gates run inside the job. +# +# SEED=0 REPO=$PWD sbatch --dependency=afterok: .../gated_mingram_seed.sbatch +# +# The gates normally run at submit time in submit_mingram.sh, which needs the tokenizers to +# exist. Two of these three are still training, so the checks move in here and run after the +# dependency clears. They are not skipped: a wrong CORE_SAFE_ARMS makes base_eval raise +# after bits-per-byte is computed but before the result block prints, losing an hour per run +# and leaving a log with no artifact_dir that every later submission retries forever. MinGram +# can also prune a token past its target, which is how fineweb_ru_5gb_plain_mingram landed at +# 34,684, so the vocabulary is checked per arm rather than assumed. +# +#SBATCH --job-name=mgds_gated +#SBATCH --partition=normal +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --time=12:00:00 + +set -euo pipefail + +: "${REPO:?set REPO to the checkout this job should run}" +cd "${REPO}" +source marker_experiments/downstream/cluster/env.sh +# Pin imports to THIS checkout. The venv installs script_bpe and pynanochat as editable +# pointers into /users/cmeister747/script_tok, and sys.path[0] is the script's directory, +# not the repo root, so without this a run resolves library code from another branch. +export PYTHONPATH="${REPO}:${REPO}/eval/py-nanochat:${PYTHONPATH:-}" + +: "${SEED:?set SEED}" +ARMS_LIST="bnd_w bnd_wpd_caps bnd_wpd_extcaps" +TRAINER=mingram +CORPUS=fineweb_en_5gb +VOCAB=34685 +DEPTH=12 +NUM_SHARDS=8 +NANOCHAT_BASE_DIR=/capstor/scratch/cscs/cmeister747/marker_downstream/nanochat_base +OUT_DIR=results/mingram_downstream + +echo "[gate] ${SLURM_JOB_ID:-nojob} seed=${SEED} node=$(hostname) commit=$(git rev-parse HEAD)" + +# 1. every tokenizer exists and is exactly the matched size. The adapter reports vocab+1 +# for the synthetic BOS it adds. +for arm in ${ARMS_LIST}; do + tok="marker_experiments/downstream/tokenizers/${CORPUS}_${arm}_${TRAINER}_v${VOCAB}.json.gz" + [[ -f "$tok" ]] || { echo "[gate] missing tokenizer: $tok" >&2; exit 1; } + n=$(uv run python -c " +from marker_experiments.downstream.boundary_tokenizer import BoundaryMinGramModel +from pynanochat.tokenizer_adapter import ScriptBPETokenizerAdapter +print(ScriptBPETokenizerAdapter(BoundaryMinGramModel.load('$tok')).get_vocab_size()) +") + if [[ "$n" != "$((VOCAB + 1))" ]]; then + echo "[gate] refusing: ${arm} vocabulary is ${n}, expected $((VOCAB + 1))" >&2 + exit 1 + fi + echo "[gate] ${arm}: vocabulary ${n}, matched, sha256=$(sha256sum "$tok" | cut -d' ' -f1)" +done + +# 2. measure the CORE prefix property and build CORE_SAFE_ARMS from the measurement. +CHECK=$(NANOCHAT_BASE="${NANOCHAT_BASE_DIR}" uv run python \ + marker_experiments/downstream/core_prefix_check.py \ + --tokenizer-dir marker_experiments/downstream/tokenizers \ + --pattern "_${TRAINER}_v${VOCAB}" --trainer "${TRAINER}" || true) +SAFE="" +for arm in ${ARMS_LIST}; do + line=$(grep -F "${CORPUS}_${arm}_${TRAINER}_v${VOCAB}.json.gz" <<< "${CHECK}" || true) + [[ -n "$line" ]] || { echo "[gate] no CORE result for ${arm}" >&2; echo "${CHECK}" >&2; exit 1; } + echo "[gate] ${line}" + [[ "$line" == *"core,bpb"* ]] && SAFE+="${arm} " +done +echo "[gate] CORE_SAFE_ARMS='${SAFE}'" + +# 3. the three runs, one per GPU, with the cores split between them. +CORES=$(nproc) +PER_RUN=$(( (CORES - 8) / 3 )) +(( PER_RUN >= 1 )) || { echo "[gate] ${PER_RUN} encode workers per run" >&2; exit 1; } +echo "[gate] 3 runs at ${PER_RUN} encode workers each" + +pids=() names=() +gpu=0 +for arm in ${ARMS_LIST}; do + ( + export CUDA_VISIBLE_DEVICES="${gpu}" + export ARMS="${arm}" SEEDS="${SEED}" + export TRAINER DEPTH CORPUS VOCAB NUM_SHARDS + export OUT="${OUT_DIR}" NANOCHAT_BASE="${NANOCHAT_BASE_DIR}" + export CORE_SAFE_ARMS="${SAFE}" + export GPUS=1 + export ENCODE_WORKERS="${PER_RUN}" TRAIN_WORKERS="${PER_RUN}" + # Pinned, not inherited from the submitting shell: a stray SMOKE=1 would train 20 + # iterations and still print a result block that collect_results.py parses into a row. + export SMOKE=0 TAG_SUFFIX= + # Those paper artifacts are tracked and carry the BPE numbers; this sweep is one trainer. + export SKIP_PAPER_ARTIFACTS=1 + marker_experiments/downstream/run_arms.sh + ) > "${OUT_DIR}/slurm/gated_${arm}_s${SEED}_${SLURM_JOB_ID:-nojob}.out" 2>&1 & + pids+=($!) names+=("${arm}/s${SEED}") + gpu=$(( gpu + 1 )) +done + +fail=0 +for i in "${!pids[@]}"; do + if wait "${pids[$i]}"; then echo "[gate] ${names[$i]} ok"; else echo "[gate] ${names[$i]} FAILED"; fail=1; fi +done +uv run python marker_experiments/downstream/collect_results.py \ + --logs-dir "${OUT_DIR}/logs" --out "${OUT_DIR}/results.tsv" || fail=1 +echo "[gate] done, fail=${fail}" +exit "${fail}" diff --git a/marker_experiments/downstream/run_arms.sh b/marker_experiments/downstream/run_arms.sh index d8039e3f..d8e1736a 100755 --- a/marker_experiments/downstream/run_arms.sh +++ b/marker_experiments/downstream/run_arms.sh @@ -71,7 +71,7 @@ echo "== step 2/3: tokenizer-side checks (must be clean before burning GPU hours # because each job is handed a per-arm manifest and so only ever sees one arm. This is the # one step in a job that loads every matched tokenizer at once. uv run python marker_experiments/downstream/smoke_test.py \ - --tokenizer-dir "$TOK_DIR" --pattern "_${TRAINER}_v${VOCAB}" \ + --tokenizer-dir "$TOK_DIR" --pattern "_${TRAINER}_v${VOCAB}" --corpus "${CORPUS}_" \ --require-matched-vocab echo "== step 3/3: downstream runs" diff --git a/marker_experiments/downstream/smoke_test.py b/marker_experiments/downstream/smoke_test.py index 30a5287a..345dd315 100755 --- a/marker_experiments/downstream/smoke_test.py +++ b/marker_experiments/downstream/smoke_test.py @@ -97,18 +97,26 @@ def fresh_process_load(path, dotted): @app.default def main(tokenizer_dir: str = DEFAULT_DIR, pattern: str = "en_", - require_matched_vocab: bool = False) -> None: + corpus: str = "", require_matched_vocab: bool = False) -> None: """Check the tokenizer-side downstream path for every matching tokenizer. Args: tokenizer_dir: Directory of .json.gz tokenizers to check. pattern: Only check files containing this substring. + corpus: Only check files for this corpus, matched as a filename prefix. Needed + because `pattern` is one substring and a filename is + {corpus}_{arm}_{trainer}_v{vocab}, so no single substring selects one + corpus and one trainer at once. --require-matched-vocab asserts that the + arms being compared share a total vocabulary, and that comparison is within + one corpus and trainer; without this filter the assertion also demands that + unrelated corpora agree, and one cell short of its target anywhere in the + directory aborts every run. """ ScriptBPETokenizerAdapter, write_token_bytes = _import_adapter() paths = sorted( os.path.join(tokenizer_dir, f) for f in os.listdir(tokenizer_dir) - if f.endswith(".json.gz") and pattern in f + if f.endswith(".json.gz") and pattern in f and f.startswith(corpus) ) if not paths: raise SystemExit(f"no tokenizers matching {pattern!r} in {tokenizer_dir}") diff --git a/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json.gz new file mode 100644 index 00000000..feb6fc3b Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json.gz differ diff --git a/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json.gz b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json.gz new file mode 100644 index 00000000..db81b185 Binary files /dev/null and b/marker_experiments/downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json.gz differ diff --git a/marker_experiments/paper/generated/manifest.json b/marker_experiments/paper/generated/manifest.json index c7969db9..da8a6ae3 100644 --- a/marker_experiments/paper/generated/manifest.json +++ b/marker_experiments/paper/generated/manifest.json @@ -308,6 +308,30 @@ "train_seconds": 460, "trainer": "mingram" }, + "fineweb_ar_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_ar_5gb_quick", + "lang": "ar", + "overshoot_factor": null, + "path": "downstream/tokenizers/fineweb_ar_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 135, + "trainer": "bpe" + }, + "fineweb_ar_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_ar_5gb_quick", + "lang": "ar", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ar_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 456, + "trainer": "mingram" + }, "fineweb_ar_5gb_quick_bnd_wpd_mingram_v34685": { "additional_vocab_size": 32974, "arm": "bnd_wpd", @@ -653,6 +677,30 @@ "train_seconds": 927, "trainer": "mingram" }, + "fineweb_de_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_de_5gb_quick", + "lang": "de", + "overshoot_factor": null, + "path": "downstream/tokenizers/fineweb_de_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 303, + "trainer": "bpe" + }, + "fineweb_de_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_de_5gb_quick", + "lang": "de", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_de_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 984, + "trainer": "mingram" + }, "fineweb_de_5gb_quick_bnd_wpd_mingram_v34685": { "additional_vocab_size": 32974, "arm": "bnd_wpd", @@ -785,6 +833,23 @@ "train_seconds": 517, "trainer": "bpe" }, + "fineweb_en_5gb_bnd_wpd_caps_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_caps", + "atomic_vocab": 1713, + "corpus": "fineweb_en_5gb", + "eval_chars": 3602925, + "eval_chars_per_token": 3.776001819393731, + "eval_slice": "marker_experiments/eval_texts/en.json", + "eval_tokens": 954164, + "lang": "en", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 867, + "trainer": "mingram" + }, "fineweb_en_5gb_bnd_wpd_extcaps_bpe_v34685": { "additional_vocab_size": 32972, "arm": "bnd_wpd_extcaps", @@ -802,6 +867,23 @@ "train_seconds": 153, "trainer": "bpe" }, + "fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcaps", + "atomic_vocab": 1713, + "corpus": "fineweb_en_5gb", + "eval_chars": 3602925, + "eval_chars_per_token": 3.7854081717348205, + "eval_slice": "marker_experiments/eval_texts/en.json", + "eval_tokens": 951793, + "lang": "en", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 876, + "trainer": "mingram" + }, "fineweb_en_5gb_bnd_wpd_mingram_v34685": { "additional_vocab_size": 32974, "arm": "bnd_wpd", @@ -1007,6 +1089,30 @@ "train_seconds": 318, "trainer": "mingram" }, + "fineweb_en_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_en_5gb_quick", + "lang": "en", + "overshoot_factor": null, + "path": "downstream/tokenizers/fineweb_en_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 80, + "trainer": "bpe" + }, + "fineweb_en_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_en_5gb_quick", + "lang": "en", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_en_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 284, + "trainer": "mingram" + }, "fineweb_en_5gb_quick_bnd_wpd_mingram_v34685": { "additional_vocab_size": 32974, "arm": "bnd_wpd", @@ -1352,6 +1458,30 @@ "train_seconds": 1692, "trainer": "mingram" }, + "fineweb_fi_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_fi_5gb_quick", + "lang": "fi", + "overshoot_factor": null, + "path": "downstream/tokenizers/fineweb_fi_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 504, + "trainer": "bpe" + }, + "fineweb_fi_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_fi_5gb_quick", + "lang": "fi", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_fi_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 1422, + "trainer": "mingram" + }, "fineweb_fi_5gb_quick_bnd_wpd_mingram_v34685": { "additional_vocab_size": 32974, "arm": "bnd_wpd", @@ -1664,6 +1794,30 @@ "train_seconds": 1498, "trainer": "mingram" }, + "fineweb_ko_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_ko_5gb_quick", + "lang": "ko", + "overshoot_factor": null, + "path": "downstream/tokenizers/fineweb_ko_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 400, + "trainer": "bpe" + }, + "fineweb_ko_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_ko_5gb_quick", + "lang": "ko", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ko_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 1504, + "trainer": "mingram" + }, "fineweb_ko_5gb_quick_bnd_wpd_mingram_v34685": { "additional_vocab_size": 32974, "arm": "bnd_wpd", @@ -2009,6 +2163,30 @@ "train_seconds": 395, "trainer": "mingram" }, + "fineweb_ru_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_ru_5gb_quick", + "lang": "ru", + "overshoot_factor": null, + "path": "downstream/tokenizers/fineweb_ru_5gb_quick_bnd_wpd_extcapsfix_bpe_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 168, + "trainer": "bpe" + }, + "fineweb_ru_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcapsfix", + "atomic_vocab": 1713, + "corpus": "fineweb_ru_5gb_quick", + "lang": "ru", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_ru_5gb_quick_bnd_wpd_extcapsfix_mingram_v34685.json.gz", + "total_vocab": 34685, + "train_seconds": 522, + "trainer": "mingram" + }, "fineweb_ru_5gb_quick_bnd_wpd_mingram_v34685": { "additional_vocab_size": 32974, "arm": "bnd_wpd", diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json new file mode 100644 index 00000000..f3349469 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_en_5gb_bnd_wpd_caps_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_caps", + "atomic_vocab": 1713, + "corpus": "fineweb_en_5gb", + "eval_chars": 3602925, + "eval_chars_per_token": 3.776001819393731, + "eval_slice": "marker_experiments/eval_texts/en.json", + "eval_tokens": 954164, + "lang": "en", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_en_5gb_bnd_wpd_caps_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 867, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json new file mode 100644 index 00000000..9fa36e99 --- /dev/null +++ b/marker_experiments/paper/generated/manifest_parts/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json @@ -0,0 +1,19 @@ +{ + "fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685": { + "additional_vocab_size": 32972, + "arm": "bnd_wpd_extcaps", + "atomic_vocab": 1713, + "corpus": "fineweb_en_5gb", + "eval_chars": 3602925, + "eval_chars_per_token": 3.7854081717348205, + "eval_slice": "marker_experiments/eval_texts/en.json", + "eval_tokens": 951793, + "lang": "en", + "overshoot_factor": 1.15, + "path": "downstream/tokenizers/fineweb_en_5gb_bnd_wpd_extcaps_mingram_v34685.json.gz", + "roundtrip_failures": 0, + "total_vocab": 34685, + "train_seconds": 876, + "trainer": "mingram" + } +} \ No newline at end of file diff --git a/marker_experiments/paper/generated/results_mingram.tsv b/marker_experiments/paper/generated/results_mingram.tsv index 18c33ccd..a917f2e1 100644 --- a/marker_experiments/paper/generated/results_mingram.tsv +++ b/marker_experiments/paper/generated/results_mingram.tsv @@ -1,4 +1,13 @@ method arm variant trainer seed val_bpb_true train_bpb_true val_bpb train_bpb byte_factor core depth vocab_size tokenizer_id log task_agi_eval_lsat_ar task_arc_challenge task_arc_easy task_bigbench_cs_algorithms task_bigbench_dyck_languages task_bigbench_language_identification task_bigbench_operators task_bigbench_qa_wikidata task_bigbench_repeat_copy_logic task_boolq task_commonsense_qa task_copa task_coqa task_hellaswag task_hellaswag_zeroshot task_jeopardy task_lambada_openai task_openbook_qa task_piqa task_squad task_winograd task_winogrande +bnd_w_mingram bnd_w mingram 0 0.8764577046320687 0.7494451294003847 1.015638 0.868456 0.862962694022938 0.1433 12 34686 bnd_w_mingram_d12_s0 bnd_w_mingram_d12_s0.log 0.070652 0.018203 0.34624 0.419697 0.154 0.178108 0.090476 0.247232 0.0 -0.17737 0.131859 0.1 0.20481 0.155547 0.159397 0.00992 0.307588 0.106667 0.350381 0.136329 0.157509 -0.014996 +bnd_w_mingram bnd_w mingram 1 0.8764749638859491 0.7490585221134624 1.015658 0.868008 0.862962694022938 0.1534 12 34686 bnd_w_mingram_d12_s1 bnd_w_mingram_d12_s1.log 0.038043 0.025028 0.371493 0.434848 0.141 0.177118 0.095238 0.267753 0.0 -0.172541 0.234234 0.18 0.181887 0.155812 0.161389 0.01795 0.285077 0.114667 0.337323 0.155345 0.135531 0.037095 +bnd_w_mingram bnd_w mingram 2 0.8760314010612215 0.7529668801546923 1.015144 0.872537 0.862962694022938 0.1359 12 34686 bnd_w_mingram_d12_s2 bnd_w_mingram_d12_s2.log 0.048913 0.048919 0.392256 0.443182 0.13 0.174147 0.104762 0.236947 0.0 -0.305328 0.035627 0.06 0.19529 0.154219 0.163646 0.010864 0.293033 0.122667 0.332971 0.174361 0.172161 0.000789 +bnd_wpd_caps_mingram bnd_wpd_caps mingram 0 0.8799920218774505 0.7564885679658033 1.044361 0.897789 0.8426128722515016 12 34686 bnd_wpd_caps_mingram_d12_s0 bnd_wpd_caps_mingram_d12_s0.log +bnd_wpd_caps_mingram bnd_wpd_caps mingram 1 0.8800434212626578 0.7557622356699226 1.044422 0.896927 0.8426128722515016 12 34686 bnd_wpd_caps_mingram_d12_s1 bnd_wpd_caps_mingram_d12_s1.log +bnd_wpd_caps_mingram bnd_wpd_caps mingram 2 0.8801934063539185 0.7600755709629781 1.0446 0.902046 0.8426128722515016 12 34686 bnd_wpd_caps_mingram_d12_s2 bnd_wpd_caps_mingram_d12_s2.log +bnd_wpd_extcaps_mingram bnd_wpd_extcaps mingram 0 0.8762511122154423 0.7531985204685082 1.03775 0.892018 0.8443759211905009 12 34686 bnd_wpd_extcaps_mingram_d12_s0 bnd_wpd_extcaps_mingram_d12_s0.log +bnd_wpd_extcaps_mingram bnd_wpd_extcaps mingram 1 0.8757926160902358 0.7527746437560705 1.037207 0.891516 0.8443759211905009 12 34686 bnd_wpd_extcaps_mingram_d12_s1 bnd_wpd_extcaps_mingram_d12_s1.log +bnd_wpd_extcaps_mingram bnd_wpd_extcaps mingram 2 0.8760830814071253 0.7556809856768083 1.037551 0.894958 0.8443759211905009 12 34686 bnd_wpd_extcaps_mingram_d12_s2 bnd_wpd_extcaps_mingram_d12_s2.log bnd_wpd_mingram bnd_wpd mingram 0 0.8797970102955008 0.756568250318537 1.044151 0.897902 0.8425955731455516 12 34686 bnd_wpd_mingram_d12_s0 bnd_wpd_mingram_d12_s0.log bnd_wpd_mingram bnd_wpd mingram 1 0.8809749589067584 0.756463768467467 1.045549 0.897778 0.8425955731455516 12 34686 bnd_wpd_mingram_d12_s1 bnd_wpd_mingram_d12_s1.log bnd_wpd_mingram bnd_wpd mingram 2 0.8807499858887284 0.7599327344421072 1.045282 0.901895 0.8425955731455516 12 34686 bnd_wpd_mingram_d12_s2 bnd_wpd_mingram_d12_s2.log diff --git a/marker_experiments/paper/generated/table_downstream_main.tex b/marker_experiments/paper/generated/table_downstream_main.tex index c7fb4d25..6e046a4b 100644 --- a/marker_experiments/paper/generated/table_downstream_main.tex +++ b/marker_experiments/paper/generated/table_downstream_main.tex @@ -10,9 +10,8 @@ & BPE & MinGram \\ \midrule \plainscheme & 0.8853 {\footnotesize $\pm$ 0.0003} & 0.8837 {\footnotesize $\pm$ 0.0008} \\ -\bnds{w} & \textbf{0.8768} {\footnotesize $\pm$ 0.0008} & -- \\ -\bnds{wpd} & 0.8800 {\footnotesize $\pm$ 0.0005} & \textbf{0.8805} {\footnotesize $\pm$ 0.0006} \\ -\bnds{wpdcaps} & \underline{0.8793} {\footnotesize $\pm$ 0.0005} & -- \\ +\bnds{w} & \textbf{0.8768} {\footnotesize $\pm$ 0.0008} & \textbf{0.8763} {\footnotesize $\pm$ 0.0003} \\ +\bnds{wpd} & \underline{0.8800} {\footnotesize $\pm$ 0.0005} & \underline{0.8805} {\footnotesize $\pm$ 0.0006} \\ \bottomrule \end{tabular} \caption{Downstream evaluation results. @@ -29,6 +28,7 @@ % CORE, depth 12, n/a wherever a scheme breaks the prefix property its tasks assume: % BPE bnd_w: 0.1411 +- 0.0058 (n=3), paired vs plain +0.0027 +- 0.0072 % BPE plain: 0.1384 +- 0.0022 (n=3) +% MinGram bnd_w: 0.1442 +- 0.0088 (n=3), paired vs plain +0.0046 +- 0.0128 % MinGram plain: 0.1396 +- 0.0061 (n=3) % raw bpb (nanochat's own figure) divides by summed token byte length and is not % comparable across these tokenizers; see the appendix tables.