Skip to content

Commit 6d1d7c9

Browse files
EssozEssoz
authored
Merge pull request #144 from OrderLab/codex/inference-identity-cache
Cache process and thread identities during precondition inference
2 parents 0866977 + 47dc072 commit 6d1d7c9

7 files changed

Lines changed: 1020 additions & 8 deletions

File tree

‎docs/measurements/inference-identity-cache-20261025.json‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
{
2+
"measurement_date": "2026-10-25",
3+
"scope": "construction plus repeated identity-checked get_meta_vars calls",
4+
"settings": {
5+
"rows": 573494,
6+
"calls": 1000,
7+
"repetitions": 3
8+
},
9+
"runtime": {
10+
"python": "3.10.16",
11+
"platform": "macOS-27.0.1-arm64-arm-64bit",
12+
"host_class": "local Darwin arm64 workstation",
13+
"logical_cpu_count": 12,
14+
"pandas": "2.2.3",
15+
"numpy": "1.26.4",
16+
"native_thread_settings": {
17+
"OMP_NUM_THREADS": "1",
18+
"OPENBLAS_NUM_THREADS": "1",
19+
"MKL_NUM_THREADS": "1"
20+
},
21+
"candidate_source_sha256": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a",
22+
"candidate_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7",
23+
"candidate_source_matches_commit": true,
24+
"legacy_methods_source_sha256": "22a7976d0fc8558b5a3dcc77e9e527572ec1f1e36b15d5e244a4a0b7ec6f44e7"
25+
},
26+
"input": {
27+
"sha256_pandas_row_hashes": "ce7af99430be7c9494ef8ee397049b77847897dd965c1082a3cda9e88a159f8b",
28+
"query_sha256": "cd7c7c854fec82747a9704c95dc1e897d6f2dca5664b1b9167a8bd845f227384",
29+
"dtypes": {
30+
"time": "int64",
31+
"process_id": "float64",
32+
"thread_id": "float64"
33+
},
34+
"dataframe_deep_bytes": 13763984,
35+
"context_manager_count": 0,
36+
"process_count": 8,
37+
"thread_count": 16
38+
},
39+
"medians": {
40+
"legacy": {
41+
"construction_seconds": 8.187499770428985e-05,
42+
"queries_seconds": 9.489749166001275,
43+
"total_seconds": 9.489831040998979
44+
},
45+
"candidate": {
46+
"construction_seconds": 5.3582996770273894e-05,
47+
"queries_seconds": 0.01196087500284193,
48+
"total_seconds": 0.01200404100382002
49+
}
50+
},
51+
"trials": [
52+
{
53+
"repetition": 1,
54+
"position": 1,
55+
"variant": "legacy",
56+
"total_seconds": 9.2562226250011
57+
},
58+
{
59+
"repetition": 1,
60+
"position": 2,
61+
"variant": "candidate",
62+
"total_seconds": 0.01413512499857461
63+
},
64+
{
65+
"repetition": 2,
66+
"position": 1,
67+
"variant": "candidate",
68+
"total_seconds": 0.010777290997793898
69+
},
70+
{
71+
"repetition": 2,
72+
"position": 2,
73+
"variant": "legacy",
74+
"total_seconds": 9.489831040998979
75+
},
76+
{
77+
"repetition": 3,
78+
"position": 1,
79+
"variant": "legacy",
80+
"total_seconds": 9.548994832999597
81+
},
82+
{
83+
"repetition": 3,
84+
"position": 2,
85+
"variant": "candidate",
86+
"total_seconds": 0.01200404100382002
87+
}
88+
],
89+
"correctness": {
90+
"all_results_equal_expected": true,
91+
"result_sha256": "481e48248fde7cbd3011bd60ecb24d4b0ccaa470a6e5a1986acf27bcc3fa9570",
92+
"input_unchanged": true
93+
},
94+
"validation": {
95+
"local_combined_tests_passed": 43,
96+
"matching_runtime_linux_identity_tests_passed": 11,
97+
"independent_differential_comparisons": 78,
98+
"independent_dtype_cases": 13,
99+
"pickle_protocols": 6,
100+
"spawn_worker_tasks_passed": 2,
101+
"matching_linux_runtime": {
102+
"python": "3.11.4",
103+
"pytorch": "2.16.0.dev20261012+cpu",
104+
"numpy": "2.1.0",
105+
"pandas": "2.3.3"
106+
}
107+
},
108+
"full_inference": {
109+
"status": "completed_unprofiled_pair",
110+
"evidence": "inference-identity-cache-full-trace-20261025.json",
111+
"baseline_wall_seconds": 1385.656381815672,
112+
"candidate_wall_seconds": 98.55111291632056,
113+
"wall_time_ratio": 14.060281419573906,
114+
"invariant_records": 835,
115+
"failed_hypothesis_records": 105,
116+
"complete_json_multisets_equal": true,
117+
"performance_comparison_valid": true,
118+
"scope": "One sequential saved-trace CPU pair with six relations enabled; no repeated-run distribution or training-overhead claim.",
119+
"profile_attempts": "Two earlier periodic-faulthandler plus cProfile attempts exited SIGSEGV without completed outputs and are excluded."
120+
},
121+
"limitations": [
122+
"Synthetic identity-lookup microbenchmark, not completed inference.",
123+
"No context-manager work or trace-file loading; input generation excluded.",
124+
"No RSS/PSS, peak-memory, or multiprocessing memory-scaling measurement.",
125+
"Cache stores O(P+T) values and returns fresh O(P+T) lists.",
126+
"Stable identity values, dtypes, and row order required within one frame."
127+
],
128+
"source_evidence_sha256": {
129+
"benchmark-full.json": "7c3b84db5645226b4d841b929185edad33c075c7868ad9951607a57b2852febe",
130+
"review-results.json": "19e4c7bf403a8a3732e8dbf0fda3ab4f6902b99c0e997dc490d9383ffc406376",
131+
"combined-tests.log": "6334a444075d2bd539f965998354922ea2d940d74536d61790e39e840bcc345c",
132+
"matching-runtime-linux-tests.log": "56b695709855bb8a17cd0fbf58e1bf3e7d17ed228719d5cc2350d3718f8e3604"
133+
}
134+
}

‎docs/measurements/inference-identity-cache-full-trace-20261025.json‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
{
2+
"measurement_date": "2026-10-25",
3+
"scope": "One baseline-first sequential unprofiled pair on the same saved training trace; no repeats, affinity, or exclusive host reservation.",
4+
"workload": {
5+
"model_parameters": 6163712,
6+
"training_steps": 10,
7+
"sequence_length": 256,
8+
"batch_size": 1,
9+
"seed": 42,
10+
"events": 573494,
11+
"raw_trace_bytes": 286335289,
12+
"training_run": "20261012T163111Z-88c4d357",
13+
"trace_files": {
14+
"trace_API_1088885_23438542501568.log": {
15+
"bytes": 82668375,
16+
"sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d"
17+
},
18+
"trace_API_1088885_23456244348224.log": {
19+
"bytes": 203666914,
20+
"sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8"
21+
}
22+
}
23+
},
24+
"settings": {
25+
"backend": "pandas",
26+
"workers": 1,
27+
"native_threads": 1,
28+
"python_hash_seed": 0,
29+
"gpu": false,
30+
"disabled_relations": [
31+
"FunctionLeadRelation",
32+
"FunctionCoverRelation"
33+
],
34+
"timeout_seconds": 1800,
35+
"sampled_child_session_pss_limit_gib": 12,
36+
"memory_sampling_seconds": 1,
37+
"profile_enabled": false,
38+
"periodic_traceback_dumps": false,
39+
"eligible_api_count": 41
40+
},
41+
"runtime": {
42+
"host": "labmach",
43+
"python": "3.11.4",
44+
"torch": "2.16.0.dev20261012+cpu",
45+
"torch_git_version": "8f9045d72f71e7aa9e16b6d7a055ec5bc1f720b5",
46+
"pandas": "2.3.3",
47+
"numpy": "2.1.0",
48+
"original_trace_runtime_difference": "Original training used Python 3.12.3 and a CUDA 13.0 Torch build at the same Torch source revision. Original eligible API name set was not logged; only its count of 41 is available."
49+
},
50+
"sources": {
51+
"baseline": "Frozen call-index snapshot underlying PR141; before publication-only typing/comment formatting changes.",
52+
"candidate_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7",
53+
"trace_pandas_sha256": {
54+
"baseline": "8e4c3589371449b78430ed5039d0159d3d0435664b218f3a96c81935be0d75d5",
55+
"candidate": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a"
56+
},
57+
"provenance_manifest_sha256": {
58+
"baseline": "7e8e267085ece31844a820752004af6a1c5bb293a3a2733990b039d72fcd872a",
59+
"candidate": "b48d705bd9dccbdf7bce17e8704e7ae5554a7d6239cfd545d7b5ffc66a888b09"
60+
},
61+
"candidate_production_delta": "Identity caching plus publication-only import/type-annotation changes, comments/docstrings and formatting; all other production behavior unchanged. Independent pruning-state fix PR145 excluded from both variants."
62+
},
63+
"comparison": {
64+
"complete_pair": true,
65+
"scope": "one sequential unprofiled run per source; no replicate/median claim",
66+
"harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777",
67+
"baseline": {
68+
"exit_code": 0,
69+
"failed_hypotheses": {
70+
"canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291",
71+
"distinct_records": 105,
72+
"records": 105
73+
},
74+
"invariants": {
75+
"canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65",
76+
"distinct_records": 835,
77+
"records": 835
78+
},
79+
"sampled_peak_concurrent_memory": {
80+
"pss_bytes": 2283671552,
81+
"rss_bytes": 2291527680,
82+
"uss_bytes": 2280935424
83+
},
84+
"source_unchanged": true,
85+
"stage_times": {
86+
"marker_durations": {
87+
"cross_trace_collection_seconds": 0.001,
88+
"generation_and_pruning_seconds": 619.58,
89+
"precondition_and_output_sort_seconds": 751.403
90+
},
91+
"markers": {
92+
"collection_start": "2026-10-25T03:37:57.781000",
93+
"generation_start": "2026-10-25T03:27:38.201000",
94+
"inference_end": "2026-10-25T03:50:29.185000",
95+
"precondition_start": "2026-10-25T03:37:57.782000"
96+
},
97+
"note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.",
98+
"reported": {
99+
"inference_seconds": 1371.0323550701141,
100+
"load_seconds": 10.503035306930542
101+
}
102+
},
103+
"stop_reason": null,
104+
"success": true,
105+
"traces_unchanged": true,
106+
"wall_seconds": 1385.656381815672
107+
},
108+
"candidate": {
109+
"exit_code": 0,
110+
"failed_hypotheses": {
111+
"canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291",
112+
"distinct_records": 105,
113+
"records": 105
114+
},
115+
"invariants": {
116+
"canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65",
117+
"distinct_records": 835,
118+
"records": 835
119+
},
120+
"sampled_peak_concurrent_memory": {
121+
"pss_bytes": 2286718976,
122+
"rss_bytes": 2294530048,
123+
"uss_bytes": 2283991040
124+
},
125+
"source_unchanged": true,
126+
"stage_times": {
127+
"marker_durations": {
128+
"cross_trace_collection_seconds": 0.0,
129+
"generation_and_pruning_seconds": 72.759,
130+
"precondition_and_output_sort_seconds": 10.913
131+
},
132+
"markers": {
133+
"collection_start": "2026-10-25T03:51:57.205000",
134+
"generation_start": "2026-10-25T03:50:44.446000",
135+
"inference_end": "2026-10-25T03:52:08.118000",
136+
"precondition_start": "2026-10-25T03:51:57.205000"
137+
},
138+
"note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.",
139+
"reported": {
140+
"inference_seconds": 83.71693992614746,
141+
"load_seconds": 10.698827981948853
142+
}
143+
},
144+
"stop_reason": null,
145+
"success": true,
146+
"traces_unchanged": true,
147+
"wall_seconds": 98.55111291632056
148+
},
149+
"invariants": {
150+
"comparison": "structural_json_multiset",
151+
"equal": true,
152+
"reference": {
153+
"records": 835,
154+
"distinct_records": 835,
155+
"canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65"
156+
},
157+
"candidate": {
158+
"records": 835,
159+
"distinct_records": 835,
160+
"canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65"
161+
},
162+
"missing_records": 0,
163+
"extra_records": 0,
164+
"missing_examples": [],
165+
"extra_examples": []
166+
},
167+
"failed_hypotheses": {
168+
"comparison": "structural_json_multiset",
169+
"equal": true,
170+
"reference": {
171+
"records": 105,
172+
"distinct_records": 105,
173+
"canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291"
174+
},
175+
"candidate": {
176+
"records": 105,
177+
"distinct_records": 105,
178+
"canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291"
179+
},
180+
"missing_records": 0,
181+
"extra_records": 0,
182+
"missing_examples": [],
183+
"extra_examples": []
184+
},
185+
"wall_time_ratio_baseline_over_candidate": 14.060281419573906
186+
},
187+
"context_index": {
188+
"baseline": [
189+
{
190+
"call": 0,
191+
"contexts": 711,
192+
"rows": 573494,
193+
"seconds": 0.46406279876828194,
194+
"states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07"
195+
},
196+
{
197+
"call": 1,
198+
"contexts": 711,
199+
"rows": 573494,
200+
"seconds": 0.4610072821378708,
201+
"states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07"
202+
}
203+
],
204+
"candidate": [
205+
{
206+
"call": 0,
207+
"contexts": 711,
208+
"rows": 573494,
209+
"seconds": 0.4815337397158146,
210+
"states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07"
211+
},
212+
{
213+
"call": 1,
214+
"contexts": 711,
215+
"rows": 573494,
216+
"seconds": 0.46630632504820824,
217+
"states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07"
218+
}
219+
]
220+
},
221+
"verification": {
222+
"complete_structural_json_multisets_equal": true,
223+
"duplicate_multiplicity_and_array_order_preserved": true,
224+
"json_object_key_order_and_record_line_order_ignored": true,
225+
"both_full_and_stage_context_files_byte_identical": true,
226+
"all_source_and_input_hashes_unchanged": true
227+
},
228+
"timer_scope": {
229+
"wall": "Child startup through exit and monitor polling, including imports and JSON serialization; initial/final integrity hashing excluded.",
230+
"load": "Read/concatenate/TracePandas construction, including harness context serialization.",
231+
"inference": "Engine construction, hypothesis generation/pruning, collection, final preconditions and sort; ends before invariant JSON serialization.",
232+
"marker_intervals": "Millisecond log timestamp resolution; generation/pruning and final preconditions/sort are parts of inference, not additional wall time."
233+
},
234+
"memory_scope": "Peak concurrent child-session samples, excluding monitor; PSS depends on shared pages. No memory improvement or multiprocessing scaling claim.",
235+
"evidence_archive": {
236+
"name": "matching-runtime-unprofiled-pair-evidence.tar.gz",
237+
"bytes": 314914,
238+
"sha256": "d78a815e1516131b28342c7d7e9251bc195f9708bab247c9f2d184558ec26f9e"
239+
},
240+
"limitations": [
241+
"One sequential pair, no distribution or confidence interval.",
242+
"Offline saved-trace CPU inference, not training, distributed scaling, online checking overhead, or new trace collection.",
243+
"Six enabled relations; FunctionLeadRelation and FunctionCoverRelation disabled identically.",
244+
"This preserves existing inference behavior; it does not certify inferred invariants as ground truth.",
245+
"Earlier periodic-faulthandler plus cProfile attempts crashed and are excluded; this successful pair alone does not isolate the crash cause."
246+
]
247+
}

0 commit comments

Comments
 (0)