Skip to content

Commit dc4bf7d

Browse files
EssozEssoz
authored
Merge pull request #140 from OrderLab/codex/inference-reservoir-cap
Bound example reservoirs across construction and inference merges
2 parents a3c7d39 + 6759bd6 commit dc4bf7d

9 files changed

Lines changed: 1173 additions & 88 deletions

‎docs/experiments/reservoir-cap-validation.json‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
{
2+
"baseline_commit": "9eb4f320d3db34d372145261704b805b88e39bda",
3+
"checkpoint_verified_date": "2026-10-25",
4+
"combined_changed_production_source_sha256": {
5+
"traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286",
6+
"traincheck/invariant/base_cls.py": "2a2092626065dd871549dd7c969f085c4bc38fe5fffd92469d2ed775bda084ae",
7+
"traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5",
8+
"traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4"
9+
},
10+
"combined_sampling_pr": 139,
11+
"combined_source_manifest_sha256": "60f431cd032532f6b3521169c29ce3ed3943c30fb5b55449ef7ebb608d6cc811",
12+
"combined_test_source_sha256": {
13+
"tests/test_example_reservoir.py": "0381a81e714980f5cff16699db951d984c4f42cd95b6fa47b1611d4d38c2f636",
14+
"tests/test_inference_sampling.py": "046554588613da74e5dbad524acd9ca9ed7eee9c96611d1341eab3cc6cd16e8e"
15+
},
16+
"controlled_probe": {
17+
"after": {
18+
"cap": 2,
19+
"constructed_num_seen": 5,
20+
"constructed_retained": 2,
21+
"merged_num_seen": 20,
22+
"merged_retained": 2
23+
},
24+
"before": {
25+
"cap": 2,
26+
"constructed_num_seen": 5,
27+
"constructed_retained": 5,
28+
"merged_num_seen": 10,
29+
"merged_retained": 4
30+
}
31+
},
32+
"limits": [
33+
"Per-occurrence marginal inclusion only for correlated source reservoirs; no joint uniformity claim.",
34+
"Optional containment generalization is skipped for already-sampled inputs; generalized hypotheses can change above the cap.",
35+
"Retained reservoirs are bounded; temporary unions and total hypothesis/process memory are not.",
36+
"Saved raw benchmark traces are external to the checked-in regression tests."
37+
],
38+
"recorded_date": "2026-10-13",
39+
"regression_tests": {
40+
"reservoir_standalone_passed": 21,
41+
"reservoir_with_sampling_passed": 21,
42+
"sampling_with_reservoir_passed": 11,
43+
"start_methods": [
44+
"fork",
45+
"spawn"
46+
],
47+
"worker_counts": [
48+
1,
49+
2,
50+
4
51+
]
52+
},
53+
"replay": {
54+
"comparison": "Full structural JSON multiset; preserves multiplicity and array order",
55+
"complete_runs": 9,
56+
"disabled_relations": [
57+
"FunctionLeadRelation",
58+
"FunctionCoverRelation"
59+
],
60+
"input_hashes_unchanged": true,
61+
"native_threads": 1,
62+
"repeats_per_workload_and_workers": 1,
63+
"runs": [
64+
{
65+
"exit_code": 0,
66+
"failed_hypotheses": {
67+
"canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820",
68+
"distinct_records": 21,
69+
"records": 21
70+
},
71+
"invariants": {
72+
"canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84",
73+
"distinct_records": 193,
74+
"records": 193
75+
},
76+
"matches_sampling_only": true,
77+
"workers": 1,
78+
"workload": "mnist"
79+
},
80+
{
81+
"exit_code": 0,
82+
"failed_hypotheses": {
83+
"canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820",
84+
"distinct_records": 21,
85+
"records": 21
86+
},
87+
"invariants": {
88+
"canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84",
89+
"distinct_records": 193,
90+
"records": 193
91+
},
92+
"matches_sampling_only": true,
93+
"workers": 2,
94+
"workload": "mnist"
95+
},
96+
{
97+
"exit_code": 0,
98+
"failed_hypotheses": {
99+
"canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820",
100+
"distinct_records": 21,
101+
"records": 21
102+
},
103+
"invariants": {
104+
"canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84",
105+
"distinct_records": 193,
106+
"records": 193
107+
},
108+
"matches_sampling_only": true,
109+
"workers": 4,
110+
"workload": "mnist"
111+
},
112+
{
113+
"exit_code": 0,
114+
"failed_hypotheses": {
115+
"canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7",
116+
"distinct_records": 10,
117+
"records": 10
118+
},
119+
"invariants": {
120+
"canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09",
121+
"distinct_records": 518,
122+
"records": 518
123+
},
124+
"matches_sampling_only": true,
125+
"workers": 1,
126+
"workload": "transformer"
127+
},
128+
{
129+
"exit_code": 0,
130+
"failed_hypotheses": {
131+
"canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7",
132+
"distinct_records": 10,
133+
"records": 10
134+
},
135+
"invariants": {
136+
"canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09",
137+
"distinct_records": 518,
138+
"records": 518
139+
},
140+
"matches_sampling_only": true,
141+
"workers": 2,
142+
"workload": "transformer"
143+
},
144+
{
145+
"exit_code": 0,
146+
"failed_hypotheses": {
147+
"canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7",
148+
"distinct_records": 10,
149+
"records": 10
150+
},
151+
"invariants": {
152+
"canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09",
153+
"distinct_records": 518,
154+
"records": 518
155+
},
156+
"matches_sampling_only": true,
157+
"workers": 4,
158+
"workload": "transformer"
159+
},
160+
{
161+
"exit_code": 0,
162+
"failed_hypotheses": {
163+
"canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204",
164+
"distinct_records": 86,
165+
"records": 86
166+
},
167+
"invariants": {
168+
"canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a",
169+
"distinct_records": 512,
170+
"records": 512
171+
},
172+
"matches_sampling_only": true,
173+
"workers": 1,
174+
"workload": "dcgan"
175+
},
176+
{
177+
"exit_code": 0,
178+
"failed_hypotheses": {
179+
"canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204",
180+
"distinct_records": 86,
181+
"records": 86
182+
},
183+
"invariants": {
184+
"canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a",
185+
"distinct_records": 512,
186+
"records": 512
187+
},
188+
"matches_sampling_only": true,
189+
"workers": 2,
190+
"workload": "dcgan"
191+
},
192+
{
193+
"exit_code": 0,
194+
"failed_hypotheses": {
195+
"canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204",
196+
"distinct_records": 86,
197+
"records": 86
198+
},
199+
"invariants": {
200+
"canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a",
201+
"distinct_records": 512,
202+
"records": 512
203+
},
204+
"matches_sampling_only": true,
205+
"workers": 4,
206+
"workload": "dcgan"
207+
}
208+
],
209+
"source_hashes_unchanged": true
210+
},
211+
"runtime": {
212+
"packages": {
213+
"numpy": "1.26.4",
214+
"pandas": "2.3.3",
215+
"torch": "2.2.2+cu121"
216+
},
217+
"platform": "Linux",
218+
"python": "3.10.16"
219+
},
220+
"saved_evidence_sha256": {
221+
"baseline-probe.json": "69d232dbeae11972ddbb9d27abb153d5f88dd0e2025417c041800583740d9289",
222+
"cap-source-manifest.json": "6910fabc4a993976f9cf90cd98d68d2797e70ff306fb5ff9d4052fab2fb2f8b1",
223+
"cap-tests.log": "7f512ee57813789d5bb907cdccbceaeea7782c95cf95b7dac62fccf9448e3a82",
224+
"combined-reservoir-tests.log": "6b0f38d6f17bafb808ba505c604ff60306a6362452233d22b6c77700a0210dbd",
225+
"combined-sampling-tests.log": "dac8a7daaff443800210eeecbadfa8b257e9cbbe7ccd30859f8951c7c0a9246c",
226+
"combined-source-manifest.json": "60f431cd032532f6b3521169c29ce3ed3943c30fb5b55449ef7ebb608d6cc811",
227+
"fixed-probe.json": "72161a6b9fe85d4b0df03f9ae4b0921bbe27def313da4ffa738fef10ca8a7304",
228+
"replay/summary.json": "aa081af685c92f9e62c3ca7f72b9082cd2ed61af7c4f1a37b543eb8e709d914c",
229+
"sampling-only-comparison.json": "e526d5cc2c9f8fa37c2aeef21a05e9481338dc142789f27edff129d3121d0a15"
230+
},
231+
"schema_version": 1,
232+
"standalone_source_sha256": {
233+
"tests/test_example_reservoir.py": "0381a81e714980f5cff16699db951d984c4f42cd95b6fa47b1611d4d38c2f636",
234+
"traincheck/infer_engine.py": "741ac7701ad4fef3089fce778ddd4d422c5245432cf582bec06095e0ed6ec71c",
235+
"traincheck/invariant/base_cls.py": "2a2092626065dd871549dd7c969f085c4bc38fe5fffd92469d2ed775bda084ae",
236+
"traincheck/invariant/contain_relation.py": "67a6613f0048542174f6655fcb1f984fabf5c5f2fa9303eab13aed93103b410d",
237+
"traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4"
238+
},
239+
"trace_inputs": {
240+
"dcgan": {
241+
"file_count": 1,
242+
"file_sha256": [
243+
"da74105c67ff699b91d96eac1fac89f3626ba9199594c9c373eb1ff6a69b15d1"
244+
],
245+
"total_bytes": 3401128
246+
},
247+
"mnist": {
248+
"file_count": 1,
249+
"file_sha256": [
250+
"6e3dd86b5ed0d7b5d556a167eaf95226852dd816a33f872668fbaf1b26638c20"
251+
],
252+
"total_bytes": 472863
253+
},
254+
"transformer": {
255+
"file_count": 1,
256+
"file_sha256": [
257+
"467e587463e72e874f5abfd14452cabdf966fd804f448eecf85f0155ea49ffb6"
258+
],
259+
"total_bytes": 1564970
260+
}
261+
}
262+
}

‎docs/experiments/reservoir-cap-validation.md‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Example reservoir cap validation
2+
3+
Iterable construction and stage/cross-trace merging previously appended retained examples directly. This bypassed `MAX_EXAMPLE_LIST_SIZE` and left the represented population count stale. The fix uses bounded insertion and merges according to the original stream populations, preserving duplicate occurrences and exact concatenation order when complete inputs fit.
4+
5+
Results below were recorded on 2026-10-13. The pending source and saved evidence hashes were independently rechecked on 2026-10-25. The baseline is `9eb4f320d3db34d372145261704b805b88e39bda`; combined validation includes the deterministic sampling change in #139.
6+
7+
## Controlled fixture
8+
9+
The fixture sets capacity to 2, merges two streams containing ten occurrences each, and separately constructs a reservoir from five occurrences.
10+
11+
| Measurement | Before | After |
12+
|---|---:|---:|
13+
| Retained after merge | 4 | 2 |
14+
| `num_seen` after merge | 10 | 20 |
15+
| Retained from five-item iterable | 5 | 2 |
16+
| `num_seen` for iterable | 5 | 5 |
17+
18+
These measure reservoir correctness. They do not measure whole-process memory savings.
19+
20+
## Regression and replay checks
21+
22+
Run the portable regression suite from the repository root:
23+
24+
```bash
25+
python -m unittest discover -s tests -p test_example_reservoir.py -v
26+
```
27+
28+
All 21 reservoir tests passed on Linux with Python 3.10.16, PyTorch 2.2.2+cu121, pandas 2.3.3, and NumPy 1.26.4. Coverage includes cap/count accounting, duplicates, atomic rejection, continuation RNG state, stage/cross-trace call sites, actual inference with 1/2/4 workers under fork and spawn, and fixed-seed marginal-inclusion sweeps. With #139, all 21 reservoir and 11 sampling tests passed; its sampling fixture explicitly disables the cap to exercise precondition downsampling separately.
29+
30+
Nine complete saved-trace inference replays used one pass per workload and worker count, with native thread counts set to 1 and `FunctionLeadRelation`/`FunctionCoverRelation` disabled. Every invariant and failed-hypothesis JSON multiset matched across workers and against sampling-only outputs; multiplicity and array order were retained in comparisons.
31+
32+
| Workload | Workers tested | Invariants | Failed hypotheses |
33+
|---|---|---:|---:|
34+
| MNIST | 1, 2, 4 | 193 | 21 |
35+
| Small transformer | 1, 2, 4 | 518 | 10 |
36+
| DCGAN | 1, 2, 4 | 512 | 86 |
37+
38+
Input and source hashes remained unchanged. Compact counts, canonical output hashes, and source/evidence hashes are recorded in [reservoir-cap-validation.json](reservoir-cap-validation.json). Its combined-source manifest and test hashes identify the validated combination with #139, including the uncapped sampling fixture. Raw saved traces are external to this note; the synthetic regression suite runs directly from the checkout.
39+
40+
Publication validation on 2026-10-25 reran all 32 combined tests successfully on macOS. Production Python ASTs match the previously validated combined source; publication changes are formatting and narrow lint comments for intentional exact-integer checks.
41+
42+
## Scope
43+
44+
The merge RNG must advance independently of source selection and must not reset across nested merges. Existing equal-seed source reservoirs can still be correlated: the guarantee is marginal occurrence inclusion, not a uniform joint sample across correlated reservoirs.
45+
46+
Optional containment generalization is skipped when an input has already been sampled, because discarded set overlap cannot be reconstructed. Complete inputs retain exact set deduplication before capping the known union. Consequently, generalized hypotheses can change above the cap. Temporary unions, total hypothesis counts, and total process memory are outside this per-reservoir bound.

0 commit comments

Comments
 (0)