{ "schema": "method-kpi-v1", "measured_at": "2026-10-27T17:32:44.409400+00:00", "evaluation_kind": "author_created_development_regression_not_independent_holdout", "summary": { "lexical": { "positive_n": 16, "top1": 1, "top5": 1, "negative_n": 4, "valid_abstentions": 4, "service_errors": 0, "median_seconds": 0.01, "p95_seconds": 0.0124, "failed_cases": [ "career", "career-plan", "product", "learning-card", "recall", "spacing", "decision", "risk", "copy", "documentation", "alternatives", "offer", "allocation", "adoption", "contribution" ] }, "hybrid": { "positive_n": 16, "top1": 16, "top5": 16, "negative_n": 4, "valid_abstentions": 4, "service_errors": 0, "median_seconds": 0.8512500000000001, "p95_seconds": 1.0014, "failed_cases": [] }, "context": { "cases": 16, "budget_checks": 48, "all_within_cap": true, "reference_retained_checks": 48, "full_top5_tokens": 29910, "compact_reference_tokens": 14924, "compact_plus_fetch_tokens": 25217, "same_answer_quality": "unmeasured", "actual_billed_usage": null }, "graph": { "methods": 55, "stable_graph": 55, "no_dangling_edges": 55, "source_links_match_record": 52, "exact_feedback_versions": 9, "source_methods": 52, "feedback_methods": 9 } }, "records": [ { "case": "career", "mode": "lexical", "expected_ids": [ "method_769519565d9e4eb68ce27c60bc879490" ], "ids": [], "available": true, "elapsed_seconds": 0.0397, "top1": false, "top5": false, "abstained": true }, { "case": "career", "mode": "hybrid", "expected_ids": [ "method_769519565d9e4eb68ce27c60bc879490" ], "ids": [ "method_769519565d9e4eb68ce27c60bc879490", "method_4409287a86e44a46ab31171d93a7b7f7", "method_62fd49754e2542da802845cb2fa8608c", "method_710ecbba8f2742ba93b4a68ff2143aae", "method_f6a1798ca0f4474589bdbd5bf74eae8a" ], "available": true, "elapsed_seconds": 1.1419, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 194, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 941, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1957, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2301, "compact_1024_tokens": 941, "compact_plus_exact_fetch_tokens": 2103, "selected_fetch_id": "method_769519565d9e4eb68ce27c60bc879490", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "career-plan", "mode": "lexical", "expected_ids": [ "method_4409287a86e44a46ab31171d93a7b7f7" ], "ids": [], "available": true, "elapsed_seconds": 0.0124, "top1": false, "top5": false, "abstained": true }, { "case": "career-plan", "mode": "hybrid", "expected_ids": [ "method_4409287a86e44a46ab31171d93a7b7f7" ], "ids": [ "method_4409287a86e44a46ab31171d93a7b7f7", "method_c3aa4545035847fbb1d95ebdce5ed1ce", "method_769519565d9e4eb68ce27c60bc879490", "method_f70995d13acc49adb28497af7bcb14a1", "method_13dad5a5dcb942c382816e96049edcf6" ], "available": true, "elapsed_seconds": 0.8497, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 211, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 1007, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1829, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2201, "compact_1024_tokens": 1007, "compact_plus_exact_fetch_tokens": 1495, "selected_fetch_id": "method_4409287a86e44a46ab31171d93a7b7f7", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "product", "mode": "lexical", "expected_ids": [ "method_710ecbba8f2742ba93b4a68ff2143aae" ], "ids": [], "available": true, "elapsed_seconds": 0.0093, "top1": false, "top5": false, "abstained": true }, { "case": "product", "mode": "hybrid", "expected_ids": [ "method_710ecbba8f2742ba93b4a68ff2143aae" ], "ids": [ "method_710ecbba8f2742ba93b4a68ff2143aae", "method_21c0995c2768476da59e7115c73205e5", "method_f70995d13acc49adb28497af7bcb14a1", "method_62fd49754e2542da802845cb2fa8608c", "method_69b5c6d56d384423a315b3c4905d81c8" ], "available": true, "elapsed_seconds": 0.9318, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 202, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 966, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 2002, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2212, "compact_1024_tokens": 966, "compact_plus_exact_fetch_tokens": 2069, "selected_fetch_id": "method_710ecbba8f2742ba93b4a68ff2143aae", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "learning-card", "mode": "lexical", "expected_ids": [ "method_21c0995c2768476da59e7115c73205e5" ], "ids": [], "available": true, "elapsed_seconds": 0.011, "top1": false, "top5": false, "abstained": true }, { "case": "learning-card", "mode": "hybrid", "expected_ids": [ "method_21c0995c2768476da59e7115c73205e5" ], "ids": [ "method_21c0995c2768476da59e7115c73205e5", "method_72c20c6ba92448e892491d9840fd691c", "method_103585915b604ffba57d547fd36da259", "method_5d0c199705e14772a33a65b55c701c5e", "method_fa79358fbb3143459eaeab135bcdb023" ], "available": true, "elapsed_seconds": 0.8913, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 198, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 957, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 2007, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2181, "compact_1024_tokens": 957, "compact_plus_exact_fetch_tokens": 1409, "selected_fetch_id": "method_21c0995c2768476da59e7115c73205e5", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "recall", "mode": "lexical", "expected_ids": [ "method_72c20c6ba92448e892491d9840fd691c" ], "ids": [], "available": true, "elapsed_seconds": 0.011, "top1": false, "top5": false, "abstained": true }, { "case": "recall", "mode": "hybrid", "expected_ids": [ "method_72c20c6ba92448e892491d9840fd691c" ], "ids": [ "method_72c20c6ba92448e892491d9840fd691c", "method_103585915b604ffba57d547fd36da259", "method_33a595e82ddc478698be3669934e6760", "method_fa79358fbb3143459eaeab135bcdb023", "method_435e3ceb63134fd887c4d9a2c722f9e5" ], "available": true, "elapsed_seconds": 0.7669, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 191, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 958, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 2001, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2182, "compact_1024_tokens": 958, "compact_plus_exact_fetch_tokens": 2032, "selected_fetch_id": "method_72c20c6ba92448e892491d9840fd691c", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "spacing", "mode": "lexical", "expected_ids": [ "method_435e3ceb63134fd887c4d9a2c722f9e5" ], "ids": [], "available": true, "elapsed_seconds": 0.01, "top1": false, "top5": false, "abstained": true }, { "case": "spacing", "mode": "hybrid", "expected_ids": [ "method_435e3ceb63134fd887c4d9a2c722f9e5" ], "ids": [ "method_435e3ceb63134fd887c4d9a2c722f9e5", "method_72c20c6ba92448e892491d9840fd691c" ], "available": true, "elapsed_seconds": 0.8024, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 201, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 968, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 968, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 854, "compact_1024_tokens": 968, "compact_plus_exact_fetch_tokens": 1438, "selected_fetch_id": "method_435e3ceb63134fd887c4d9a2c722f9e5", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "decision", "mode": "lexical", "expected_ids": [ "method_aa62abb9e67e47488f69c1251cd43304" ], "ids": [], "available": true, "elapsed_seconds": 0.0093, "top1": false, "top5": false, "abstained": true }, { "case": "decision", "mode": "hybrid", "expected_ids": [ "method_aa62abb9e67e47488f69c1251cd43304" ], "ids": [ "method_aa62abb9e67e47488f69c1251cd43304", "method_70ceb860f1514053bb8314db63a326ef", "method_9210344a5c964d4a94657848b2ce3f8f", "method_c3aa4545035847fbb1d95ebdce5ed1ce", "method_d1d75cc1bc94400c9560324063ec5287" ], "available": true, "elapsed_seconds": 0.7483, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 207, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 910, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1742, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2224, "compact_1024_tokens": 910, "compact_plus_exact_fetch_tokens": 1403, "selected_fetch_id": "method_aa62abb9e67e47488f69c1251cd43304", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "risk", "mode": "lexical", "expected_ids": [ "method_f70995d13acc49adb28497af7bcb14a1" ], "ids": [], "available": true, "elapsed_seconds": 0.0115, "top1": false, "top5": false, "abstained": true }, { "case": "risk", "mode": "hybrid", "expected_ids": [ "method_f70995d13acc49adb28497af7bcb14a1" ], "ids": [ "method_f70995d13acc49adb28497af7bcb14a1", "method_710ecbba8f2742ba93b4a68ff2143aae", "method_62fd49754e2542da802845cb2fa8608c", "method_f6a1798ca0f4474589bdbd5bf74eae8a" ], "available": true, "elapsed_seconds": 0.7709, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 194, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 974, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1710, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 1794, "compact_1024_tokens": 974, "compact_plus_exact_fetch_tokens": 1449, "selected_fetch_id": "method_f70995d13acc49adb28497af7bcb14a1", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "copy", "mode": "lexical", "expected_ids": [ "method_33a595e82ddc478698be3669934e6760" ], "ids": [], "available": true, "elapsed_seconds": 0.0117, "top1": false, "top5": false, "abstained": true }, { "case": "copy", "mode": "hybrid", "expected_ids": [ "method_33a595e82ddc478698be3669934e6760" ], "ids": [ "method_33a595e82ddc478698be3669934e6760", "method_fa79358fbb3143459eaeab135bcdb023", "method_103585915b604ffba57d547fd36da259", "method_5d0c199705e14772a33a65b55c701c5e", "method_0b8450340161413d96cd38d879e84ae3" ], "available": true, "elapsed_seconds": 0.9044, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 199, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 894, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1575, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2287, "compact_1024_tokens": 894, "compact_plus_exact_fetch_tokens": 1364, "selected_fetch_id": "method_33a595e82ddc478698be3669934e6760", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "documentation", "mode": "lexical", "expected_ids": [ "method_103585915b604ffba57d547fd36da259" ], "ids": [], "available": true, "elapsed_seconds": 0.0092, "top1": false, "top5": false, "abstained": true }, { "case": "documentation", "mode": "hybrid", "expected_ids": [ "method_103585915b604ffba57d547fd36da259" ], "ids": [ "method_103585915b604ffba57d547fd36da259", "method_33a595e82ddc478698be3669934e6760", "method_fa79358fbb3143459eaeab135bcdb023", "method_72c20c6ba92448e892491d9840fd691c" ], "available": true, "elapsed_seconds": 0.8727, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 197, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 983, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1654, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 1757, "compact_1024_tokens": 983, "compact_plus_exact_fetch_tokens": 1461, "selected_fetch_id": "method_103585915b604ffba57d547fd36da259", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "alternatives", "mode": "lexical", "expected_ids": [ "method_c3aa4545035847fbb1d95ebdce5ed1ce" ], "ids": [], "available": true, "elapsed_seconds": 0.0096, "top1": false, "top5": false, "abstained": true }, { "case": "alternatives", "mode": "hybrid", "expected_ids": [ "method_c3aa4545035847fbb1d95ebdce5ed1ce" ], "ids": [ "method_c3aa4545035847fbb1d95ebdce5ed1ce", "method_769519565d9e4eb68ce27c60bc879490", "method_21c0995c2768476da59e7115c73205e5", "method_f6a1798ca0f4474589bdbd5bf74eae8a", "method_ca9a0e9cfe634cba9c28b0b086b12670" ], "available": true, "elapsed_seconds": 0.8528, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 200, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 997, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1828, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2217, "compact_1024_tokens": 997, "compact_plus_exact_fetch_tokens": 1447, "selected_fetch_id": "method_c3aa4545035847fbb1d95ebdce5ed1ce", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "offer", "mode": "lexical", "expected_ids": [ "method_db0a40d480064c61bd720e62b232ca5d" ], "ids": [], "available": true, "elapsed_seconds": 0.0094, "top1": false, "top5": false, "abstained": true }, { "case": "offer", "mode": "hybrid", "expected_ids": [ "method_db0a40d480064c61bd720e62b232ca5d" ], "ids": [ "method_db0a40d480064c61bd720e62b232ca5d", "method_c3aa4545035847fbb1d95ebdce5ed1ce" ], "available": true, "elapsed_seconds": 0.8271, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 202, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 737, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 737, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 828, "compact_1024_tokens": 737, "compact_plus_exact_fetch_tokens": 1203, "selected_fetch_id": "method_db0a40d480064c61bd720e62b232ca5d", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "allocation", "mode": "lexical", "expected_ids": [ "method_ca9a0e9cfe634cba9c28b0b086b12670" ], "ids": [], "available": true, "elapsed_seconds": 0.0108, "top1": false, "top5": false, "abstained": true }, { "case": "allocation", "mode": "hybrid", "expected_ids": [ "method_ca9a0e9cfe634cba9c28b0b086b12670" ], "ids": [ "method_ca9a0e9cfe634cba9c28b0b086b12670", "method_7c5dc3d99bfc4e03ba08ebe209259bae", "method_f040265d2ae640c49d8614c218e06d61" ], "available": true, "elapsed_seconds": 0.9954, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 201, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 954, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1446, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 1433, "compact_1024_tokens": 954, "compact_plus_exact_fetch_tokens": 1463, "selected_fetch_id": "method_ca9a0e9cfe634cba9c28b0b086b12670", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "costs", "mode": "lexical", "expected_ids": [ "method_7c5dc3d99bfc4e03ba08ebe209259bae" ], "ids": [ "method_7c5dc3d99bfc4e03ba08ebe209259bae" ], "available": true, "elapsed_seconds": 0.0096, "top1": true, "top5": true, "abstained": false }, { "case": "costs", "mode": "hybrid", "expected_ids": [ "method_7c5dc3d99bfc4e03ba08ebe209259bae" ], "ids": [ "method_7c5dc3d99bfc4e03ba08ebe209259bae", "method_ca9a0e9cfe634cba9c28b0b086b12670" ], "available": true, "elapsed_seconds": 0.8981, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 202, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 828, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1059, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 934, "compact_1024_tokens": 828, "compact_plus_exact_fetch_tokens": 2017, "selected_fetch_id": "method_7c5dc3d99bfc4e03ba08ebe209259bae", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "adoption", "mode": "lexical", "expected_ids": [ "method_62fd49754e2542da802845cb2fa8608c" ], "ids": [], "available": true, "elapsed_seconds": 0.0121, "top1": false, "top5": false, "abstained": true }, { "case": "adoption", "mode": "hybrid", "expected_ids": [ "method_62fd49754e2542da802845cb2fa8608c" ], "ids": [ "method_62fd49754e2542da802845cb2fa8608c", "method_7c5dc3d99bfc4e03ba08ebe209259bae", "method_d340976ef2f543a2a168b66f51c10beb", "method_78c6bc2939a645c1a02c4cae35c0c79a", "method_13dad5a5dcb942c382816e96049edcf6" ], "available": true, "elapsed_seconds": 0.8974, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 197, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 954, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1911, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2347, "compact_1024_tokens": 954, "compact_plus_exact_fetch_tokens": 1478, "selected_fetch_id": "method_62fd49754e2542da802845cb2fa8608c", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "contribution", "mode": "lexical", "expected_ids": [ "method_f6a1798ca0f4474589bdbd5bf74eae8a" ], "ids": [], "available": true, "elapsed_seconds": 0.0119, "top1": false, "top5": false, "abstained": true }, { "case": "contribution", "mode": "hybrid", "expected_ids": [ "method_f6a1798ca0f4474589bdbd5bf74eae8a" ], "ids": [ "method_f6a1798ca0f4474589bdbd5bf74eae8a", "method_f70995d13acc49adb28497af7bcb14a1", "method_681dc89fa9bc4ccb9ead50121fe5b0aa", "method_4ae1a991a2954dc2b7c8e30465a08215", "method_9210344a5c964d4a94657848b2ce3f8f" ], "available": true, "elapsed_seconds": 1.0014, "top1": true, "top5": true, "abstained": false, "context": { "budgets": [ { "cap": 256, "actual_tokens": 201, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 1024, "actual_tokens": 896, "counter_matches": true, "within_cap": true, "expected_reference_retained": true }, { "cap": 2048, "actual_tokens": 1690, "counter_matches": true, "within_cap": true, "expected_reference_retained": true } ], "full_top5_method_tokens": 2158, "compact_1024_tokens": 896, "compact_plus_exact_fetch_tokens": 1386, "selected_fetch_id": "method_f6a1798ca0f4474589bdbd5bf74eae8a", "limits": "o200k JSON material counts, not billed usage. Reference pack requires exact fetch; preserving an expected ID is not answer sufficiency." } }, { "case": "negative-medical", "mode": "lexical", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.0092, "top1": false, "top5": false, "abstained": true }, { "case": "negative-medical", "mode": "hybrid", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.7631, "top1": false, "top5": false, "abstained": true }, { "case": "negative-return", "mode": "lexical", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.01, "top1": false, "top5": false, "abstained": true }, { "case": "negative-return", "mode": "hybrid", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.801, "top1": false, "top5": false, "abstained": true }, { "case": "negative-legal", "mode": "lexical", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.0095, "top1": false, "top5": false, "abstained": true }, { "case": "negative-legal", "mode": "hybrid", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.7557, "top1": false, "top5": false, "abstained": true }, { "case": "negative-science", "mode": "lexical", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.008, "top1": false, "top5": false, "abstained": true }, { "case": "negative-science", "mode": "hybrid", "expected_ids": [], "ids": [], "available": true, "elapsed_seconds": 0.7094, "top1": false, "top5": false, "abstained": true } ], "graph_audits": [ { "id": "method_f6a1798ca0f4474589bdbd5bf74eae8a", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0025, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_62fd49754e2542da802845cb2fa8608c", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_7c5dc3d99bfc4e03ba08ebe209259bae", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 2 }, { "id": "method_ca9a0e9cfe634cba9c28b0b086b12670", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0034, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_db0a40d480064c61bd720e62b232ca5d", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0028, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_c3aa4545035847fbb1d95ebdce5ed1ce", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_103585915b604ffba57d547fd36da259", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0024, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_33a595e82ddc478698be3669934e6760", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_f70995d13acc49adb28497af7bcb14a1", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_aa62abb9e67e47488f69c1251cd43304", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_435e3ceb63134fd887c4d9a2c722f9e5", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0017, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_72c20c6ba92448e892491d9840fd691c", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 2 }, { "id": "method_21c0995c2768476da59e7115c73205e5", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_710ecbba8f2742ba93b4a68ff2143aae", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 2 }, { "id": "method_4409287a86e44a46ab31171d93a7b7f7", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0035, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_769519565d9e4eb68ce27c60bc879490", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0019, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 2 }, { "id": "method_5b230aa120c54d198e86887fcf8a4ab3", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_13dad5a5dcb942c382816e96049edcf6", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0026, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 2 }, { "id": "method_d210e9033f2042a9b17537de7d701bd0", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0021, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_5d0c199705e14772a33a65b55c701c5e", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0021, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_fa79358fbb3143459eaeab135bcdb023", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_0b8450340161413d96cd38d879e84ae3", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 2 }, { "id": "method_5af91cc7219a4130a3a2cf2d7630b031", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0026, "node_count": 8, "edge_count": 8, "graph_kind": "record_relationships", "source_nodes_checked": 3, "feedback_nodes_checked": 2 }, { "id": "method_9317be20ea014745ad2797501d8071f2", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0026, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_1f1e279f40a341be94d39582c3f07bf3", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0031, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_f2369e7d6c8f4b0c8106488e0d045040", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0041, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_afc7efba9f844983991fa3d189d590f5", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0026, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_c3fb38e44f2443feb02cfd3564d96fc3", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.002, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_d340976ef2f543a2a168b66f51c10beb", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_78c6bc2939a645c1a02c4cae35c0c79a", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0024, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_769e62c4f39a41fea7b3e75b03a144d5", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0024, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_8e25f26e4dc04645baac2d888405da77", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0021, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_fe795a3d5f804d128698b245460e110f", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_681dc89fa9bc4ccb9ead50121fe5b0aa", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0025, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_bcc11858c6c9496aa4960f750a0e846a", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0021, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_b7bf97be541c4646b582b31f21416147", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_70ceb860f1514053bb8314db63a326ef", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0033, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_e554a06582d14c8f99683436ae00fcba", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0025, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_4ae1a991a2954dc2b7c8e30465a08215", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0024, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_17f6da889fe34a25bd84a860ab77007a", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0025, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_d1d75cc1bc94400c9560324063ec5287", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.004, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_252f680bb20944a190aa1d1355e6b5d5", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0022, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_64c6d70e0ab547c29e3eb9bb9a98d0f6", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_f040265d2ae640c49d8614c218e06d61", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_a0cc103673c54d9a89a76f7218576a77", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0038, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_bab61e9f0dab4d7cb2b91e32f2fc64c7", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0024, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_f1e336e8600447259c14626ef98584d8", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0025, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 2 }, { "id": "method_8be94f157fba42ec95ee906e4b53ae03", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 5, "edge_count": 4, "graph_kind": "record_relationships", "source_nodes_checked": 2, "feedback_nodes_checked": 0 }, { "id": "method_8bd35edb30bc4bc6b425113fa28d4320", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0035, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_6562f35901794b5e9f8f477796e7505b", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0019, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_9210344a5c964d4a94657848b2ce3f8f", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.002, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_69b5c6d56d384423a315b3c4905d81c8", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0019, "node_count": 4, "edge_count": 3, "graph_kind": "record_relationships", "source_nodes_checked": 1, "feedback_nodes_checked": 0 }, { "id": "method_5fed1c9bbb4b493f85f9ae4127f28d6c", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0039, "node_count": 3, "edge_count": 2, "graph_kind": "record_relationships", "source_nodes_checked": 0, "feedback_nodes_checked": 0 }, { "id": "method_ecbf46a7f49149cda31dd6d01799cf4f", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.002, "node_count": 3, "edge_count": 2, "graph_kind": "record_relationships", "source_nodes_checked": 0, "feedback_nodes_checked": 0 }, { "id": "method_d6bab8275ed741ae959436712962e7ce", "version": 1, "stable_graph": true, "no_dangling_edges": true, "source_links_match_record": true, "exact_feedback_versions": true, "graph_seconds": 0.0023, "node_count": 6, "edge_count": 6, "graph_kind": "record_relationships", "source_nodes_checked": 0, "feedback_nodes_checked": 3 } ], "unmeasured": [ "human_preference", "revision_time", "causal_answer_quality_gain", "business_value", "independent_OSS_reproduction" ], "limits": "Retrieval of expected methods and record integrity are measured. Source truth, expert identity, execution success, and answer quality are not inferred.", "input_hashes": { "queries": "6949dd037cfef66eb420a5b68203cdb2d2143ab6c2ab92523ab9c5731bb3f8bf", "mapping": "477fc709e69cf2887303c1a87fb931808c0d601271c3a177e75b4e9c205d3e22" }, "incremental_index": { "methods": 55, "embedded_fields": 0, "reused_fields": 110, "model": "Qwen/Qwen3-Embedding-0.6B", "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", "embedding_spec": "openakashic-methods-qwen3-v1-lasttoken-l2-f32-max512", "index_unchanged": true }, "environment": "WSL loopback · frozen public corpus of 55 methods", "raw_run_sha256": "5d0881465cb7ffc1aecc7bfc85c669da51a2fb0a34a1136cd350ea581228a33c", "denominator_note": "Source/version checks exclude methods with no corresponding source/feedback nodes. Derived from the same frozen source snapshot; raw run unchanged.", "legacy_regression": { "schema": "prior-holdout-reused-as-regression-v1", "at": "2026-10-27T17:40:40.316286+00:00", "gold_sha256": "271581334583ea2f8ec7c8d75fdbf7e3f51b487193483b1f0a9a2707485e3780", "limits": "Previously exposed 2026-10-27 holdout reused on expanded 55-method pool. Not a fresh independent holdout; no retuning on these results.", "summary": { "lexical": { "positive_n": 12, "top1": 0, "top5": 0, "no_material_n": 4, "valid_abstentions": 4, "conflict_n": 4, "excluded_absent": 4, "service_errors": 0, "failures": [ { "case": "hold-a01", "type": "answerable", "titles": [] }, { "case": "hold-a02", "type": "answerable", "titles": [] }, { "case": "hold-a03", "type": "answerable", "titles": [] }, { "case": "hold-a04", "type": "answerable", "titles": [] }, { "case": "hold-a05", "type": "answerable", "titles": [] }, { "case": "hold-a06", "type": "answerable", "titles": [] }, { "case": "hold-a07", "type": "answerable", "titles": [] }, { "case": "hold-a08", "type": "answerable", "titles": [] }, { "case": "hold-a09", "type": "answerable", "titles": [] }, { "case": "hold-a10", "type": "answerable", "titles": [] }, { "case": "hold-a11", "type": "answerable", "titles": [] }, { "case": "hold-a12", "type": "answerable", "titles": [] } ] }, "hybrid": { "positive_n": 12, "top1": 11, "top5": 12, "no_material_n": 4, "valid_abstentions": 0, "conflict_n": 4, "excluded_absent": 4, "service_errors": 0, "failures": [ { "case": "hold-n01", "type": "no_material", "titles": [ "설명문을 독자 과제·개념·참조로 나눠 쓰기", "입력 오류를 필드와 연결하고 복구 방법 전달하기" ] }, { "case": "hold-n02", "type": "no_material", "titles": [ "실험 결과에서 관찰·해석·다음 행동 분리하기", "입력 오류를 필드와 연결하고 복구 방법 전달하기" ] }, { "case": "hold-n03", "type": "no_material", "titles": [ "설명문을 독자 과제·개념·참조로 나눠 쓰기" ] }, { "case": "hold-n04", "type": "no_material", "titles": [ "SQLite 온라인 백업 및 복원본 검증" ] } ] } }, "records": [ { "case": "hold-a01", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "research-01-quality-screen" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a01", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "research-01-quality-screen" ], "keys": [ "research-01-quality-screen", "collaboration-09-feedback-tracking", "research-10-intake-check", "research-09-sensitivity", "method_5b230aa120c54d198e86887fcf8a4ab3" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "외부 데이터 품질 선별", "리뷰 피드백 대응 추적", "데이터 인수 체크리스트", "결과 민감도 비교표", "입력 오류를 필드와 연결하고 복구 방법 전달하기" ] }, { "case": "hold-a02", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "research-03-missing-policy" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a02", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "research-03-missing-policy" ], "keys": [ "research-09-sensitivity", "research-03-missing-policy", "research-04-csv-schema", "method_5b230aa120c54d198e86887fcf8a4ab3", "research-05-time-series-merge" ], "top1": false, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "결과 민감도 비교표", "결측값 처리 정책 비교", "CSV 수집 스키마 점검", "입력 오류를 필드와 연결하고 복구 방법 전달하기", "다중 파일 시계열 통합" ] }, { "case": "hold-a03", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "research-04-csv-schema" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a03", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "research-04-csv-schema" ], "keys": [ "research-04-csv-schema", "research-05-time-series-merge" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "CSV 수집 스키마 점검", "다중 파일 시계열 통합" ] }, { "case": "hold-a04", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "research-05-time-series-merge" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a04", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "research-05-time-series-merge" ], "keys": [ "research-05-time-series-merge" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "다중 파일 시계열 통합" ] }, { "case": "hold-a05", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "collaboration-05-source-ledger" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a05", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "collaboration-05-source-ledger" ], "keys": [ "collaboration-05-source-ledger", "research-10-intake-check", "research-01-quality-screen", "collaboration-07-version-anchor", "method_72c20c6ba92448e892491d9840fd691c" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "자료 출처 표와 요약 메모", "데이터 인수 체크리스트", "외부 데이터 품질 선별", "협업 문서에 이름 있는 기준 버전 기록", "읽은 내용을 가리고 회상한 뒤 오답 교정하기" ] }, { "case": "hold-a06", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "collaboration-09-feedback-tracking" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a06", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "collaboration-09-feedback-tracking" ], "keys": [ "collaboration-09-feedback-tracking", "collaboration-08-pull-request-review", "collaboration-07-version-anchor", "collaboration-03-comment-actions", "method_5b230aa120c54d198e86887fcf8a4ab3" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "리뷰 피드백 대응 추적", "PR 변경사항 검토", "협업 문서에 이름 있는 기준 버전 기록", "검토 댓글을 실행 항목으로 전환", "입력 오류를 필드와 연결하고 복구 방법 전달하기" ] }, { "case": "hold-a07", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "research-07-category-map" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a07", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "research-07-category-map" ], "keys": [ "research-07-category-map", "research-04-csv-schema", "collaboration-08-pull-request-review" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "범주값 정규화 감사", "CSV 수집 스키마 점검", "PR 변경사항 검토" ] }, { "case": "hold-a08", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "collaboration-01-document-diff" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a08", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "collaboration-01-document-diff" ], "keys": [ "collaboration-01-document-diff", "collaboration-04-change-note", "collaboration-07-version-anchor", "collaboration-09-feedback-tracking", "research-04-csv-schema" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "두 문서 변경점 검토", "기존 수정본 차이 설명 작성", "협업 문서에 이름 있는 기준 버전 기록", "리뷰 피드백 대응 추적", "CSV 수집 스키마 점검" ] }, { "case": "hold-a09", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "collaboration-02-suggested-edits" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a09", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "collaboration-02-suggested-edits" ], "keys": [ "collaboration-02-suggested-edits", "collaboration-08-pull-request-review", "collaboration-01-document-diff", "collaboration-09-feedback-tracking", "method_13dad5a5dcb942c382816e96049edcf6" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "문서 제안 편집 검토", "PR 변경사항 검토", "두 문서 변경점 검토", "리뷰 피드백 대응 추적", "확정 전 요약 확인과 값이 보존되는 수정 경로 만들기" ] }, { "case": "hold-a10", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "operations-10-sqlite-backup" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a10", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "operations-10-sqlite-backup" ], "keys": [ "operations-10-sqlite-backup" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "SQLite 온라인 백업 및 복원본 검증" ] }, { "case": "hold-a11", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "collaboration-03-comment-actions" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a11", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "collaboration-03-comment-actions" ], "keys": [ "collaboration-03-comment-actions", "collaboration-10-action-register", "collaboration-09-feedback-tracking", "collaboration-07-version-anchor", "collaboration-02-suggested-edits" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "검토 댓글을 실행 항목으로 전환", "팀 지식 문서에서 조치 목록 만들기", "리뷰 피드백 대응 추적", "협업 문서에 이름 있는 기준 버전 기록", "문서 제안 편집 검토" ] }, { "case": "hold-a12", "type": "answerable", "mode": "lexical", "available": true, "expected": [ "collaboration-07-version-anchor" ], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-a12", "type": "answerable", "mode": "hybrid", "available": true, "expected": [ "collaboration-07-version-anchor" ], "keys": [ "collaboration-07-version-anchor", "collaboration-09-feedback-tracking", "collaboration-08-pull-request-review" ], "top1": true, "top5": true, "valid_abstention": false, "excluded_absent": true, "titles": [ "협업 문서에 이름 있는 기준 버전 기록", "리뷰 피드백 대응 추적", "PR 변경사항 검토" ] }, { "case": "hold-n01", "type": "no_material", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-n01", "type": "no_material", "mode": "hybrid", "available": true, "expected": [], "keys": [ "method_103585915b604ffba57d547fd36da259", "method_5b230aa120c54d198e86887fcf8a4ab3" ], "top1": false, "top5": false, "valid_abstention": false, "excluded_absent": true, "titles": [ "설명문을 독자 과제·개념·참조로 나눠 쓰기", "입력 오류를 필드와 연결하고 복구 방법 전달하기" ] }, { "case": "hold-n02", "type": "no_material", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-n02", "type": "no_material", "mode": "hybrid", "available": true, "expected": [], "keys": [ "method_21c0995c2768476da59e7115c73205e5", "method_5b230aa120c54d198e86887fcf8a4ab3" ], "top1": false, "top5": false, "valid_abstention": false, "excluded_absent": true, "titles": [ "실험 결과에서 관찰·해석·다음 행동 분리하기", "입력 오류를 필드와 연결하고 복구 방법 전달하기" ] }, { "case": "hold-n03", "type": "no_material", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-n03", "type": "no_material", "mode": "hybrid", "available": true, "expected": [], "keys": [ "method_103585915b604ffba57d547fd36da259" ], "top1": false, "top5": false, "valid_abstention": false, "excluded_absent": true, "titles": [ "설명문을 독자 과제·개념·참조로 나눠 쓰기" ] }, { "case": "hold-n04", "type": "no_material", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-n04", "type": "no_material", "mode": "hybrid", "available": true, "expected": [], "keys": [ "operations-10-sqlite-backup" ], "top1": false, "top5": false, "valid_abstention": false, "excluded_absent": false, "titles": [ "SQLite 온라인 백업 및 복원본 검증" ] }, { "case": "hold-c01", "type": "constraint_conflict", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-c01", "type": "constraint_conflict", "mode": "hybrid", "available": true, "expected": [], "keys": [ "research-04-csv-schema", "method_d6bab8275ed741ae959436712962e7ce", "method_5fed1c9bbb4b493f85f9ae4127f28d6c", "operations-10-sqlite-backup" ], "top1": false, "top5": false, "valid_abstention": false, "excluded_absent": true, "titles": [ "CSV 수집 스키마 점검", "공개 예제 · 오프라인 문서 검색에서 원문 ID와 인용 보존", "공개 예제 · 오프라인 문서 수량 집계", "SQLite 온라인 백업 및 복원본 검증" ] }, { "case": "hold-c02", "type": "constraint_conflict", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-c02", "type": "constraint_conflict", "mode": "hybrid", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-c03", "type": "constraint_conflict", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-c03", "type": "constraint_conflict", "mode": "hybrid", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-c04", "type": "constraint_conflict", "mode": "lexical", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] }, { "case": "hold-c04", "type": "constraint_conflict", "mode": "hybrid", "available": true, "expected": [], "keys": [], "top1": false, "top5": false, "valid_abstention": true, "excluded_absent": true, "titles": [] } ] }, "public_projection": "All case IDs, expected/returned IDs, failures, graph audits and measured summaries retained. Duplicated method bodies omitted; public fixture supplies method content. Original raw run hash is retained.", "context_selection_note": "The expected method ID selects the additional exact fetch (oracle assumption); no independent agent choice or same-quality answer evaluation.", "navigation_workload": { "schema": "method-navigation-workload-v1", "measured_at": "2026-10-28T02:08:25.475143+00:00", "method_count": 55, "case_definition_sha256": "2b19f89b592ae642e582f3e8ee3a3c47c3d32f9e057af5c2ba2ec41bb555a225", "cases": { "sources": [ { "source_url": "https://docs.github.com/en/actions/concepts/workflows-and-actions/deployment-environments", "expected_refs": [ [ "method_769e62c4f39a41fea7b3e75b03a144d5", 1 ], [ "method_8e25f26e4dc04645baac2d888405da77", 1 ] ] }, { "source_url": "https://docs.github.com/en/pull-requests/how-tos/review-pull-requests", "expected_refs": [ [ "method_681dc89fa9bc4ccb9ead50121fe5b0aa", 1 ], [ "method_bcc11858c6c9496aa4960f750a0e846a", 1 ] ] }, { "source_url": "https://learn.microsoft.com/en-us/azure/well-architected/design-guides/incident-management", "expected_refs": [ [ "method_1f1e279f40a341be94d39582c3f07bf3", 1 ], [ "method_afc7efba9f844983991fa3d189d590f5", 1 ], [ "method_c3fb38e44f2443feb02cfd3564d96fc3", 1 ], [ "method_f2369e7d6c8f4b0c8106488e0d045040", 1 ] ] }, { "source_url": "https://pandas.pydata.org/docs/user_guide/io.html", "expected_refs": [ [ "method_8bd35edb30bc4bc6b425113fa28d4320", 1 ], [ "method_8be94f157fba42ec95ee906e4b53ae03", 1 ] ] }, { "source_url": "https://pandas.pydata.org/docs/user_guide/missing_data.html", "expected_refs": [ [ "method_6562f35901794b5e9f8f477796e7505b", 1 ], [ "method_f040265d2ae640c49d8614c218e06d61", 1 ] ] }, { "source_url": "https://raw.githubusercontent.com/anthropics/skills/41bbe19d1a1a7eaab5e7bb9050a417e5c6cffc8f/skills/frontend-design/SKILL.md", "expected_refs": [ [ "method_0b8450340161413d96cd38d879e84ae3", 1 ], [ "method_fa79358fbb3143459eaeab135bcdb023", 1 ] ] }, { "source_url": "https://support.google.com/docs/answer/18512262?hl=en", "expected_refs": [ [ "method_252f680bb20944a190aa1d1355e6b5d5", 1 ], [ "method_4ae1a991a2954dc2b7c8e30465a08215", 1 ] ] }, { "source_url": "https://support.google.com/docs/answer/6033474?hl=en", "expected_refs": [ [ "method_70ceb860f1514053bb8314db63a326ef", 1 ], [ "method_d1d75cc1bc94400c9560324063ec5287", 1 ] ] }, { "source_url": "https://support.google.com/docs/answer/65129", "expected_refs": [ [ "method_17f6da889fe34a25bd84a860ab77007a", 1 ], [ "method_70ceb860f1514053bb8314db63a326ef", 1 ], [ "method_fe795a3d5f804d128698b245460e110f", 1 ] ] }, { "source_url": "https://www.pon.harvard.edu/tag/reservation-price/", "expected_refs": [ [ "method_c3aa4545035847fbb1d95ebdce5ed1ce", 1 ], [ "method_db0a40d480064c61bd720e62b232ca5d", 1 ] ] }, { "source_url": "https://www.usgs.gov/data-management/data-management-plans", "expected_refs": [ [ "method_64c6d70e0ab547c29e3eb9bb9a98d0f6", 1 ], [ "method_9210344a5c964d4a94657848b2ce3f8f", 1 ], [ "method_a0cc103673c54d9a89a76f7218576a77", 1 ], [ "method_e554a06582d14c8f99683436ae00fcba", 1 ] ] }, { "source_url": "https://www.usgs.gov/data-management/manage-quality", "expected_refs": [ [ "method_64c6d70e0ab547c29e3eb9bb9a98d0f6", 1 ], [ "method_69b5c6d56d384423a315b3c4905d81c8", 1 ] ] } ], "feedback": [ { "id": "method_7c5dc3d99bfc4e03ba08ebe209259bae", "version": 1, "expected_outcomes": [ "outcome_4282d88416604c6b8ff91ef44cdb7a62" ], "expected_opinions": [ "opinion_a4c4912a27ef4d3a82b2c3f210b3b597" ] }, { "id": "method_72c20c6ba92448e892491d9840fd691c", "version": 1, "expected_outcomes": [ "outcome_e18cab1c16bf4423b9539fe6349b88a1" ], "expected_opinions": [ "opinion_b11904b5623445ef8bdf2b1c2e677274" ] }, { "id": "method_710ecbba8f2742ba93b4a68ff2143aae", "version": 1, "expected_outcomes": [ "outcome_ba90f8aa272641918e1869ff6be96612" ], "expected_opinions": [ "opinion_678d5208c4b841e6aed8d5631096f336" ] }, { "id": "method_769519565d9e4eb68ce27c60bc879490", "version": 1, "expected_outcomes": [ "outcome_714e5173989748499b84f50607e134a7" ], "expected_opinions": [ "opinion_3d2d2bd53a2946daa72c769e83581657" ] }, { "id": "method_13dad5a5dcb942c382816e96049edcf6", "version": 1, "expected_outcomes": [ "outcome_13c1969e05df4e978539754e81f2d2fe" ], "expected_opinions": [ "opinion_467999ba755f40e8ad39e83f88a7dc4d" ] }, { "id": "method_0b8450340161413d96cd38d879e84ae3", "version": 1, "expected_outcomes": [ "outcome_749327aa53bd4c2fba1b5daf731765ae" ], "expected_opinions": [ "opinion_9c9534851b4343439b02fc58960e097b" ] }, { "id": "method_5af91cc7219a4130a3a2cf2d7630b031", "version": 1, "expected_outcomes": [ "outcome_6e0f30e335424b3bbb3d7a5fe2fa7d5b" ], "expected_opinions": [ "opinion_8a18d628cf444a008bb03b533bae1d85" ] }, { "id": "method_f1e336e8600447259c14626ef98584d8", "version": 1, "expected_outcomes": [ "outcome_cef93b1a927845fc9afb584287a5de09" ], "expected_opinions": [ "opinion_03336be9d3734faa8761398a4ffc4fc6" ] }, { "id": "method_d6bab8275ed741ae959436712962e7ce", "version": 1, "expected_outcomes": [ "outcome_3fbe8ba7e67048a0b998857c154b9e48", "outcome_d54adcdb893348b4a598ac6312b50c49" ], "expected_opinions": [ "opinion_156b41db3c554bcaba8e36672fa12e9e" ] } ] }, "records": [ { "kind": "shared_source", "source_url": "https://docs.github.com/en/actions/concepts/workflows-and-actions/deployment-environments", "focus": "source_95202ecc27a4c598d7e6c880", "expected_refs": [ [ "method_769e62c4f39a41fea7b3e75b03a144d5", 1 ], [ "method_8e25f26e4dc04645baac2d888405da77", 1 ] ], "returned_refs": [ [ "method_769e62c4f39a41fea7b3e75b03a144d5", 1 ], [ "method_8e25f26e4dc04645baac2d888405da77", 1 ] ], "success": true, "graph_tokens": 783, "flat_index_lookup_tokens": 104, "elapsed_seconds": 0.027974 }, { "kind": "shared_source", "source_url": "https://docs.github.com/en/pull-requests/how-tos/review-pull-requests", "focus": "source_8df6d27d3ff51caa9e87347f", "expected_refs": [ [ "method_681dc89fa9bc4ccb9ead50121fe5b0aa", 1 ], [ "method_bcc11858c6c9496aa4960f750a0e846a", 1 ] ], "returned_refs": [ [ "method_681dc89fa9bc4ccb9ead50121fe5b0aa", 1 ], [ "method_bcc11858c6c9496aa4960f750a0e846a", 1 ] ], "success": true, "graph_tokens": 764, "flat_index_lookup_tokens": 99, "elapsed_seconds": 0.025905 }, { "kind": "shared_source", "source_url": "https://learn.microsoft.com/en-us/azure/well-architected/design-guides/incident-management", "focus": "source_bdbe392cbedf10b0ace87fe9", "expected_refs": [ [ "method_1f1e279f40a341be94d39582c3f07bf3", 1 ], [ "method_afc7efba9f844983991fa3d189d590f5", 1 ], [ "method_c3fb38e44f2443feb02cfd3564d96fc3", 1 ], [ "method_f2369e7d6c8f4b0c8106488e0d045040", 1 ] ], "returned_refs": [ [ "method_1f1e279f40a341be94d39582c3f07bf3", 1 ], [ "method_afc7efba9f844983991fa3d189d590f5", 1 ], [ "method_c3fb38e44f2443feb02cfd3564d96fc3", 1 ], [ "method_f2369e7d6c8f4b0c8106488e0d045040", 1 ] ], "success": true, "graph_tokens": 1201, "flat_index_lookup_tokens": 186, "elapsed_seconds": 0.026599 }, { "kind": "shared_source", "source_url": "https://pandas.pydata.org/docs/user_guide/io.html", "focus": "source_650c69742cd6264fcf44c4e7", "expected_refs": [ [ "method_8bd35edb30bc4bc6b425113fa28d4320", 1 ], [ "method_8be94f157fba42ec95ee906e4b53ae03", 1 ] ], "returned_refs": [ [ "method_8bd35edb30bc4bc6b425113fa28d4320", 1 ], [ "method_8be94f157fba42ec95ee906e4b53ae03", 1 ] ], "success": true, "graph_tokens": 791, "flat_index_lookup_tokens": 97, "elapsed_seconds": 0.027207 }, { "kind": "shared_source", "source_url": "https://pandas.pydata.org/docs/user_guide/missing_data.html", "focus": "source_11a0ca3244a7f1ff1521293e", "expected_refs": [ [ "method_6562f35901794b5e9f8f477796e7505b", 1 ], [ "method_f040265d2ae640c49d8614c218e06d61", 1 ] ], "returned_refs": [ [ "method_6562f35901794b5e9f8f477796e7505b", 1 ], [ "method_f040265d2ae640c49d8614c218e06d61", 1 ] ], "success": true, "graph_tokens": 782, "flat_index_lookup_tokens": 96, "elapsed_seconds": 0.026673 }, { "kind": "shared_source", "source_url": "https://raw.githubusercontent.com/anthropics/skills/41bbe19d1a1a7eaab5e7bb9050a417e5c6cffc8f/skills/frontend-design/SKILL.md", "focus": "source_b532586070a17007864f3cbd", "expected_refs": [ [ "method_0b8450340161413d96cd38d879e84ae3", 1 ], [ "method_fa79358fbb3143459eaeab135bcdb023", 1 ] ], "returned_refs": [ [ "method_0b8450340161413d96cd38d879e84ae3", 1 ], [ "method_fa79358fbb3143459eaeab135bcdb023", 1 ] ], "success": true, "graph_tokens": 816, "flat_index_lookup_tokens": 144, "elapsed_seconds": 0.026647 }, { "kind": "shared_source", "source_url": "https://support.google.com/docs/answer/18512262?hl=en", "focus": "source_42070f1fa34103dfdf8720ea", "expected_refs": [ [ "method_252f680bb20944a190aa1d1355e6b5d5", 1 ], [ "method_4ae1a991a2954dc2b7c8e30465a08215", 1 ] ], "returned_refs": [ [ "method_252f680bb20944a190aa1d1355e6b5d5", 1 ], [ "method_4ae1a991a2954dc2b7c8e30465a08215", 1 ] ], "success": true, "graph_tokens": 765, "flat_index_lookup_tokens": 101, "elapsed_seconds": 0.026048 }, { "kind": "shared_source", "source_url": "https://support.google.com/docs/answer/6033474?hl=en", "focus": "source_5e10193afa0d5636dd3d0f4a", "expected_refs": [ [ "method_70ceb860f1514053bb8314db63a326ef", 1 ], [ "method_d1d75cc1bc94400c9560324063ec5287", 1 ] ], "returned_refs": [ [ "method_70ceb860f1514053bb8314db63a326ef", 1 ], [ "method_d1d75cc1bc94400c9560324063ec5287", 1 ] ], "success": true, "graph_tokens": 781, "flat_index_lookup_tokens": 94, "elapsed_seconds": 0.02633 }, { "kind": "shared_source", "source_url": "https://support.google.com/docs/answer/65129", "focus": "source_3dbcb0b24a1a5f9e4a1239b1", "expected_refs": [ [ "method_17f6da889fe34a25bd84a860ab77007a", 1 ], [ "method_70ceb860f1514053bb8314db63a326ef", 1 ], [ "method_fe795a3d5f804d128698b245460e110f", 1 ] ], "returned_refs": [ [ "method_17f6da889fe34a25bd84a860ab77007a", 1 ], [ "method_70ceb860f1514053bb8314db63a326ef", 1 ], [ "method_fe795a3d5f804d128698b245460e110f", 1 ] ], "success": true, "graph_tokens": 1007, "flat_index_lookup_tokens": 132, "elapsed_seconds": 0.027542 }, { "kind": "shared_source", "source_url": "https://www.pon.harvard.edu/tag/reservation-price/", "focus": "source_258846853091d479723b395e", "expected_refs": [ [ "method_c3aa4545035847fbb1d95ebdce5ed1ce", 1 ], [ "method_db0a40d480064c61bd720e62b232ca5d", 1 ] ], "returned_refs": [ [ "method_c3aa4545035847fbb1d95ebdce5ed1ce", 1 ], [ "method_db0a40d480064c61bd720e62b232ca5d", 1 ] ], "success": true, "graph_tokens": 763, "flat_index_lookup_tokens": 114, "elapsed_seconds": 0.028896 }, { "kind": "shared_source", "source_url": "https://www.usgs.gov/data-management/data-management-plans", "focus": "source_2656007ac256d98530608d14", "expected_refs": [ [ "method_64c6d70e0ab547c29e3eb9bb9a98d0f6", 1 ], [ "method_9210344a5c964d4a94657848b2ce3f8f", 1 ], [ "method_a0cc103673c54d9a89a76f7218576a77", 1 ], [ "method_e554a06582d14c8f99683436ae00fcba", 1 ] ], "returned_refs": [ [ "method_64c6d70e0ab547c29e3eb9bb9a98d0f6", 1 ], [ "method_9210344a5c964d4a94657848b2ce3f8f", 1 ], [ "method_a0cc103673c54d9a89a76f7218576a77", 1 ], [ "method_e554a06582d14c8f99683436ae00fcba", 1 ] ], "success": true, "graph_tokens": 1172, "flat_index_lookup_tokens": 177, "elapsed_seconds": 0.028713 }, { "kind": "shared_source", "source_url": "https://www.usgs.gov/data-management/manage-quality", "focus": "source_50d04904abc3d27a9f34ee06", "expected_refs": [ [ "method_64c6d70e0ab547c29e3eb9bb9a98d0f6", 1 ], [ "method_69b5c6d56d384423a315b3c4905d81c8", 1 ] ], "returned_refs": [ [ "method_64c6d70e0ab547c29e3eb9bb9a98d0f6", 1 ], [ "method_69b5c6d56d384423a315b3c4905d81c8", 1 ] ], "success": true, "graph_tokens": 801, "flat_index_lookup_tokens": 100, "elapsed_seconds": 0.025898 }, { "kind": "feedback", "method_id": "method_7c5dc3d99bfc4e03ba08ebe209259bae", "expected_outcomes": [ "outcome_4282d88416604c6b8ff91ef44cdb7a62" ], "returned_outcomes": [ "outcome_4282d88416604c6b8ff91ef44cdb7a62" ], "expected_opinions": [ "opinion_a4c4912a27ef4d3a82b2c3f210b3b597" ], "returned_opinions": [ "opinion_a4c4912a27ef4d3a82b2c3f210b3b597" ], "success": true, "graph_tokens": 1102, "direct_method_tokens": 1189, "flat_projection_tokens": 787, "elapsed_seconds": 0.002383 }, { "kind": "feedback", "method_id": "method_72c20c6ba92448e892491d9840fd691c", "expected_outcomes": [ "outcome_e18cab1c16bf4423b9539fe6349b88a1" ], "returned_outcomes": [ "outcome_e18cab1c16bf4423b9539fe6349b88a1" ], "expected_opinions": [ "opinion_b11904b5623445ef8bdf2b1c2e677274" ], "returned_opinions": [ "opinion_b11904b5623445ef8bdf2b1c2e677274" ], "success": true, "graph_tokens": 1035, "direct_method_tokens": 1074, "flat_projection_tokens": 695, "elapsed_seconds": 0.002167 }, { "kind": "feedback", "method_id": "method_710ecbba8f2742ba93b4a68ff2143aae", "expected_outcomes": [ "outcome_ba90f8aa272641918e1869ff6be96612" ], "returned_outcomes": [ "outcome_ba90f8aa272641918e1869ff6be96612" ], "expected_opinions": [ "opinion_678d5208c4b841e6aed8d5631096f336" ], "returned_opinions": [ "opinion_678d5208c4b841e6aed8d5631096f336" ], "success": true, "graph_tokens": 1035, "direct_method_tokens": 1103, "flat_projection_tokens": 711, "elapsed_seconds": 0.002068 }, { "kind": "feedback", "method_id": "method_769519565d9e4eb68ce27c60bc879490", "expected_outcomes": [ "outcome_714e5173989748499b84f50607e134a7" ], "returned_outcomes": [ "outcome_714e5173989748499b84f50607e134a7" ], "expected_opinions": [ "opinion_3d2d2bd53a2946daa72c769e83581657" ], "returned_opinions": [ "opinion_3d2d2bd53a2946daa72c769e83581657" ], "success": true, "graph_tokens": 1024, "direct_method_tokens": 1162, "flat_projection_tokens": 717, "elapsed_seconds": 0.001828 }, { "kind": "feedback", "method_id": "method_13dad5a5dcb942c382816e96049edcf6", "expected_outcomes": [ "outcome_13c1969e05df4e978539754e81f2d2fe" ], "returned_outcomes": [ "outcome_13c1969e05df4e978539754e81f2d2fe" ], "expected_opinions": [ "opinion_467999ba755f40e8ad39e83f88a7dc4d" ], "returned_opinions": [ "opinion_467999ba755f40e8ad39e83f88a7dc4d" ], "success": true, "graph_tokens": 1501, "direct_method_tokens": 1833, "flat_projection_tokens": 1451, "elapsed_seconds": 0.002112 }, { "kind": "feedback", "method_id": "method_0b8450340161413d96cd38d879e84ae3", "expected_outcomes": [ "outcome_749327aa53bd4c2fba1b5daf731765ae" ], "returned_outcomes": [ "outcome_749327aa53bd4c2fba1b5daf731765ae" ], "expected_opinions": [ "opinion_9c9534851b4343439b02fc58960e097b" ], "returned_opinions": [ "opinion_9c9534851b4343439b02fc58960e097b" ], "success": true, "graph_tokens": 1525, "direct_method_tokens": 1880, "flat_projection_tokens": 1450, "elapsed_seconds": 0.002255 }, { "kind": "feedback", "method_id": "method_5af91cc7219a4130a3a2cf2d7630b031", "expected_outcomes": [ "outcome_6e0f30e335424b3bbb3d7a5fe2fa7d5b" ], "returned_outcomes": [ "outcome_6e0f30e335424b3bbb3d7a5fe2fa7d5b" ], "expected_opinions": [ "opinion_8a18d628cf444a008bb03b533bae1d85" ], "returned_opinions": [ "opinion_8a18d628cf444a008bb03b533bae1d85" ], "success": true, "graph_tokens": 1454, "direct_method_tokens": 1620, "flat_projection_tokens": 957, "elapsed_seconds": 0.002573 }, { "kind": "feedback", "method_id": "method_f1e336e8600447259c14626ef98584d8", "expected_outcomes": [ "outcome_cef93b1a927845fc9afb584287a5de09" ], "returned_outcomes": [ "outcome_cef93b1a927845fc9afb584287a5de09" ], "expected_opinions": [ "opinion_03336be9d3734faa8761398a4ffc4fc6" ], "returned_opinions": [ "opinion_03336be9d3734faa8761398a4ffc4fc6" ], "success": true, "graph_tokens": 1139, "direct_method_tokens": 1213, "flat_projection_tokens": 767, "elapsed_seconds": 0.00215 }, { "kind": "feedback", "method_id": "method_d6bab8275ed741ae959436712962e7ce", "expected_outcomes": [ "outcome_3fbe8ba7e67048a0b998857c154b9e48", "outcome_d54adcdb893348b4a598ac6312b50c49" ], "returned_outcomes": [ "outcome_3fbe8ba7e67048a0b998857c154b9e48", "outcome_d54adcdb893348b4a598ac6312b50c49" ], "expected_opinions": [ "opinion_156b41db3c554bcaba8e36672fa12e9e" ], "returned_opinions": [ "opinion_156b41db3c554bcaba8e36672fa12e9e" ], "success": true, "graph_tokens": 1178, "direct_method_tokens": 1671, "flat_projection_tokens": 1112, "elapsed_seconds": 0.021379 } ], "summary": { "source_tasks": 12, "source_success": 12, "feedback_tasks": 9, "feedback_success": 9, "shared_graph_tokens": 10426, "flat_index_once_tokens": 3545, "flat_lookup_tokens": 1444, "all_full_records_once_tokens": 35441, "feedback_graph_tokens": 10993, "feedback_direct_tokens": 12745, "feedback_flat_projection_tokens": 8647, "median_graph_seconds": 0.025905 }, "limits": [ "Exhaustive tasks generated from this stored corpus, not independent natural-language questions.", "Deterministic URL and record ID lookup; no model generated or graded answers.", "The flat URL index uses the exact same stored references and is a strong baseline. It can be reused.", "Counts are o200k JSON transport material, not billed usage or latency of model reasoning.", "Graph versus direct record material includes different structural fields; no claim of equal answer quality.", "Human navigation speed, novel task output quality and independent OSS adoption remain unmeasured." ] }, "output_quality_comparison": { "source_report": "/research/methods/design-lab/results.json", "generated_at": "2026-10-27T02:39:48.329488+00:00", "synthetic_tasks": 2, "conditions": [ "unaided", "direct", "retrieval" ], "checks": { "unaided": { "exhibition": { "passed": 7, "total": 7, "checks": { "content_fidelity": true, "visit_action_scrolls": true, "three_disclosures_toggle": true, "no_overflow_390_and_1440": true, "keyboard_operable": true, "no_console_error_or_external_request": true, "no_false_claim_or_payment": true }, "note": "세 해설의 키보드 개폐·안내 이동, 필수 내용, 390/1440px을 확인. 외부 요청은 제한 CSP와 코드 점검.", "method": "Parent browser observations plus explicit content/code inspection. Not user evaluation or WCAG conformance." }, "workshop": { "passed": 12, "total": 12, "checks": { "content_and_minimum_fields": true, "missing_input_blocks": true, "invalid_email_blocks": true, "out_of_range_people_blocks": true, "total_for_three_84000": true, "review_before_confirmation": true, "edit_preserves_values": true, "mock_confirmation_with_summary": true, "no_overflow_390_and_1440": true, "controls_labeled_and_keyboard_operable": true, "errors_associated_and_focus_recoverable": true, "no_persistence_network_or_false_booking": true }, "note": "누락·이메일·인원 오류, 3명 84,000원, 확인·수정·모의 완료, 초기화를 확인. 보조기술 전수 검사는 아님.", "method": "Parent browser observations plus explicit content/code inspection. Not user evaluation or WCAG conformance." } }, "direct": { "exhibition": { "passed": 7, "total": 7, "checks": { "content_fidelity": true, "visit_action_scrolls": true, "three_disclosures_toggle": true, "no_overflow_390_and_1440": true, "keyboard_operable": true, "no_console_error_or_external_request": true, "no_false_claim_or_payment": true }, "note": "세 해설의 키보드 개폐·안내 이동, 필수 내용, 390/1440px을 확인. 외부 요청은 제한 CSP와 코드 점검.", "method": "Parent browser observations plus explicit content/code inspection. Not user evaluation or WCAG conformance." }, "workshop": { "passed": 12, "total": 12, "checks": { "content_and_minimum_fields": true, "missing_input_blocks": true, "invalid_email_blocks": true, "out_of_range_people_blocks": true, "total_for_three_84000": true, "review_before_confirmation": true, "edit_preserves_values": true, "mock_confirmation_with_summary": true, "no_overflow_390_and_1440": true, "controls_labeled_and_keyboard_operable": true, "errors_associated_and_focus_recoverable": true, "no_persistence_network_or_false_booking": true }, "note": "누락·이메일·인원 오류, 3명 84,000원, 확인·수정·모의 완료, 초기화를 확인. 보조기술 전수 검사는 아님.", "method": "Parent browser observations plus explicit content/code inspection. Not user evaluation or WCAG conformance." } }, "retrieval": { "exhibition": { "passed": 7, "total": 7, "checks": { "content_fidelity": true, "visit_action_scrolls": true, "three_disclosures_toggle": true, "no_overflow_390_and_1440": true, "keyboard_operable": true, "no_console_error_or_external_request": true, "no_false_claim_or_payment": true }, "note": "세 해설의 키보드 개폐·안내 이동, 필수 내용, 390/1440px을 확인. 외부 요청은 제한 CSP와 코드 점검.", "method": "Parent browser observations plus explicit content/code inspection. Not user evaluation or WCAG conformance." }, "workshop": { "passed": 12, "total": 12, "checks": { "content_and_minimum_fields": true, "missing_input_blocks": true, "invalid_email_blocks": true, "out_of_range_people_blocks": true, "total_for_three_84000": true, "review_before_confirmation": true, "edit_preserves_values": true, "mock_confirmation_with_summary": true, "no_overflow_390_and_1440": true, "controls_labeled_and_keyboard_operable": true, "errors_associated_and_focus_recoverable": true, "no_persistence_network_or_false_booking": true }, "note": "누락·이메일·인원 오류, 3명 84,000원, 확인·수정·모의 완료, 초기화를 확인. 보조기술 전수 검사는 아님.", "method": "Parent browser observations plus explicit content/code inspection. Not user evaluation or WCAG conformance." } } }, "finding": "All three conditions passed exhibition 7/7 and workshop 12/12. No functional advantage observed. Model visual review included ties; not human preference or causal quality evidence.", "new_generation_in_this_run": false, "billed_usage": null }, "current_verdict": { "graph_navigation": "available", "method_comparison_and_brief": "available", "general_output_quality_superiority": "not_demonstrated", "human_utility": "unmeasured", "actual_billed_savings": "unmeasured" }, "application_evaluation": { "schema": "atlas-bounded-application-evaluation-v1", "generated_at": "2026-10-28T03:14:39.408783+00:00", "status": "complete", "protocol_sha256": "5d6bb8e2f982d1183e23d7c5f25592dcb22b690763f4e62c50c1444fbca6acbb", "design": { "tasks": 6, "outputs": 18, "generation_model": "gpt-6-luna", "review_model": "gpt-6-sol", "replicates_per_task_condition": 1, "blind_review": "condition labels hidden; task-wise shuffled answer IDs; answer content can reveal source availability", "baseline": "same complete public records as flat leaf-path JSON vs nested compact JSON; unaided third arm", "task_selection": "synthetic corpus-aligned, manually chosen records before generation; T2 adjacent product-experiment evidence only", "initial_fixture_error": "Korean stdin corrupted before generation; ASCII v2 frozen and used for all arms; initial artifacts retained as rejected audit inputs" }, "summary": { "unaided": { "criterion_passes_model_reviewed": 24, "criterion_possible": 24, "judgment_total": 35, "judgment_possible": 36, "word_limit_passes": 6, "outputs": 6, "constraint_violations": [], "unsupported_claims": [ { "task_id": "T4", "items": [ "Repeats the unsubstantiated superlative 'The most innovative learning!' from the draft." ] } ] }, "flat_same_records": { "criterion_passes_model_reviewed": 23, "criterion_possible": 24, "judgment_total": 35, "judgment_possible": 36, "word_limit_passes": 3, "outputs": 6, "constraint_violations": [ { "task_id": "T2", "items": [ "Question 3 presupposes an expectation mismatch; question 4 presupposes stopping." ] } ], "unsupported_claims": [] }, "compact_same_records": { "criterion_passes_model_reviewed": 23, "criterion_possible": 24, "judgment_total": 35, "judgment_possible": 36, "word_limit_passes": 6, "outputs": 6, "constraint_violations": [ { "task_id": "T3", "items": [ "Allows the Approver to defer choosing an option on Friday if a fact is missing, despite the fixed choice requirement." ] } ], "unsupported_claims": [] } }, "material": { "input_estimates": { "unaided-input.json": { "utf8_bytes": 2617, "characters": 2617, "tokens_o200k_base": 597, "sha256": "9ce9eef736ae40a03259ae05a69e17c136fbb34426ed74fe77f6b26a77ea5ee4" }, "flat_same_records-input.json": { "utf8_bytes": 24600, "characters": 20064, "tokens_o200k_base": 6857, "sha256": "bc315347521f935e3da7e76bcdf835d5548a58ad9a91e2691fb82cf63cdce82a" }, "compact_same_records-input.json": { "utf8_bytes": 23967, "characters": 19431, "tokens_o200k_base": 5959, "sha256": "9bfbab44ce12b829a74111743915d3962fba96999efd391b0f98662d3ca03565" } }, "compact_vs_flat_token_estimate_reduction": 0.13096106168878519, "actual_provider_usage": null, "actual_cost": null, "timing_comparison": null, "interpretation": "Artifact token estimates only, not billed/provider tokens or costs." }, "by_condition": { "unaided": [ { "blind_id": "T1-B", "objective_checks": [ { "check": "30-minute allocation", "pass": true, "evidence": "Allocates all 30 minutes across five time windows." }, { "check": "hypothesis versus evidence", "pass": true, "evidence": "Separates observed metrics and log inspection from any cause conclusion." }, { "check": "recovery or stop criterion", "pass": true, "evidence": "Verifies error rate and core flows; escalates if recovery is not evident." }, { "check": "no data deletion", "pass": true, "evidence": "Explicitly prohibits production-data deletion." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 174, "word_limit_pass": true, "notes": "Clear mitigation thresholds and post-change check.", "task_id": "T1" }, { "blind_id": "T2-B", "objective_checks": [ { "check": "five questions", "pass": true, "evidence": "Contains exactly five numbered interview questions." }, { "check": "specific past behavior", "pass": true, "evidence": "Starts with the last reservation attempt and asks what happened next." }, { "check": "20-minute allocation", "pass": true, "evidence": "Allocates 2+15+3 minutes per interview." }, { "check": "observation versus inference", "pass": true, "evidence": "Separates factual summaries, observed evidence, and interpretation." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 167, "word_limit_pass": true, "notes": "Concise consent, recording, and comparison plan.", "task_id": "T2" }, { "blind_id": "T3-A", "objective_checks": [ { "check": "one decision owner", "pass": true, "evidence": "Names a single decision owner." }, { "check": "Friday deadline", "pass": true, "evidence": "Requires the owner to choose and communicate by Friday." }, { "check": "decision evidence", "pass": true, "evidence": "Uses existing design, engineering, and sales evidence with assumptions labeled." }, { "check": "consultation versus decision", "pass": true, "evidence": "Others present evidence and objections; owner chooses without unanimity." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 169, "word_limit_pass": true, "notes": "Useful Wednesday/Thursday/Friday cadence and reversible fallback.", "task_id": "T3" }, { "blind_id": "T4-A", "objective_checks": [ { "check": "November 3 and 14:00", "pass": true, "evidence": "Includes November 3 at 14:00." }, { "check": "Civic Hall floor 2", "pass": true, "evidence": "Includes Civic Hall, floor 2." }, { "check": "free and capacity 20", "pass": true, "evidence": "States free admission and capacity of 20." }, { "check": "October 31 and phone", "pass": true, "evidence": "Gives October 31 deadline and registration phone number." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 1, "constraint_violations": [], "unsupported_claims": [ "Repeats the unsubstantiated superlative 'The most innovative learning!' from the draft." ], "word_count": 35, "word_limit_pass": true, "notes": "All logistics are intact; promotional superlative remains unsupported.", "task_id": "T4" }, { "blind_id": "T5-B", "objective_checks": [ { "check": "seven-day schedule", "pass": true, "evidence": "Lists Day 1 through Day 7." }, { "check": "20 minutes daily", "pass": true, "evidence": "States 20 minutes daily and 140 total." }, { "check": "closed-book recall", "pass": true, "evidence": "Requires closed-book memory retrieval and mixed tests." }, { "check": "check and correct errors", "pass": true, "evidence": "Directs checking and correcting each session, with a weak-item list." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 152, "word_limit_pass": true, "notes": "Practical schedule, though per-day subtask minutes are not broken down.", "task_id": "T5" }, { "blind_id": "T6-A", "objective_checks": [ { "check": "read contribution rules", "pass": true, "evidence": "Directs reading the contribution guide and development instructions." }, { "check": "search existing discussions", "pass": true, "evidence": "Directs searching prior issues and pull requests." }, { "check": "reproduction or measurement", "pass": true, "evidence": "Calls for reproducible fixture, environment, baseline method, and tests." }, { "check": "confirm scope before submission", "pass": true, "evidence": "Asks a focused question before coding if goal is unclear and checks compatibility and scope before PR." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 176, "word_limit_pass": true, "notes": "No PR is submitted; behavior preservation and measurement limits are explicit.", "task_id": "T6" } ], "flat_same_records": [ { "blind_id": "T1-A", "objective_checks": [ { "check": "30-minute allocation", "pass": true, "evidence": "Uses 0-5, 5-12, 12-20, and 20-30 minute windows." }, { "check": "hypothesis versus evidence", "pass": true, "evidence": "Says not to assume a cause and reads logs before inferring one." }, { "check": "recovery or stop criterion", "pass": true, "evidence": "Requires stable metrics and working key flows before confirming recovery; escalates if worsening." }, { "check": "no data deletion", "pass": true, "evidence": "Explicitly says not to delete production data." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 206, "word_limit_pass": true, "notes": "Specific, reversible incident decision plan with owner authorization and monitoring.", "task_id": "T1" }, { "blind_id": "T2-C", "objective_checks": [ { "check": "five questions", "pass": true, "evidence": "Contains exactly five numbered interview questions." }, { "check": "specific past behavior", "pass": true, "evidence": "Asks about the most recent attempt and its action sequence." }, { "check": "20-minute allocation", "pass": true, "evidence": "Allocates 2+12+4+2 minutes per interview." }, { "check": "observation versus inference", "pass": true, "evidence": "Explicitly distinguishes observed behavior from interpretation and notes limited inference." } ], "objective_total": 4, "method_fit": 1, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [ "Question 3 presupposes an expectation mismatch; question 4 presupposes stopping." ], "unsupported_claims": [], "word_count": 224, "word_limit_pass": false, "notes": "Questions are mostly episode based, but two prompts may steer the account.", "task_id": "T2" }, { "blind_id": "T3-C", "objective_checks": [ { "check": "one decision owner", "pass": true, "evidence": "Names one Approver with actual authority." }, { "check": "Friday deadline", "pass": true, "evidence": "Calls for a Friday review and decision." }, { "check": "decision evidence", "pass": true, "evidence": "Gathers existing funnel, support/sales, effort, risk, and design evidence." }, { "check": "consultation versus decision", "pass": true, "evidence": "Contributors advise while Approver makes the choice despite disagreement." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 232, "word_limit_pass": false, "notes": "Explicit fallback chooses a reversible option or retains current flow.", "task_id": "T3" }, { "blind_id": "T4-B", "objective_checks": [ { "check": "November 3 and 14:00", "pass": true, "evidence": "Includes November 3 at 14:00." }, { "check": "Civic Hall floor 2", "pass": true, "evidence": "Includes Civic Hall, floor 2." }, { "check": "free and capacity 20", "pass": true, "evidence": "States free admission and capacity of 20." }, { "check": "October 31 and phone", "pass": true, "evidence": "Gives October 31 deadline and registration phone number." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 28, "word_limit_pass": true, "notes": "Short, direct registration instructions with all supplied facts.", "task_id": "T4" }, { "blind_id": "T5-A", "objective_checks": [ { "check": "seven-day schedule", "pass": true, "evidence": "Lists Day 1 through Day 7." }, { "check": "20 minutes daily", "pass": true, "evidence": "Each day sums to 20 minutes; states 140 minutes total." }, { "check": "closed-book recall", "pass": true, "evidence": "Uses memory-first explanations and closed-book assessments." }, { "check": "check and correct errors", "pass": true, "evidence": "Checks against course material and tracks/corrects gaps." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 207, "word_limit_pass": true, "notes": "Detailed time budget and final score denominator.", "task_id": "T5" }, { "blind_id": "T6-B", "objective_checks": [ { "check": "read contribution rules", "pass": true, "evidence": "Directs reading the contribution guide and related documentation." }, { "check": "search existing discussions", "pass": true, "evidence": "Directs checking issues, discussions, changes, and pull requests." }, { "check": "reproduction or measurement", "pass": true, "evidence": "Calls for local reproduction and repeated baseline timings." }, { "check": "confirm scope before submission", "pass": false, "evidence": "Does not actually confirm proposed change scope with maintainers before submission; only prepares an unposted question and checks format." } ], "objective_total": 3, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 226, "word_limit_pass": false, "notes": "Strong preparation and measurement plan; maintainer scope confirmation remains implicit.", "task_id": "T6" } ], "compact_same_records": [ { "blind_id": "T1-C", "objective_checks": [ { "check": "30-minute allocation", "pass": true, "evidence": "Allocates 0-5, 5-12, 12-20, and 20-30 minutes." }, { "check": "hypothesis versus evidence", "pass": true, "evidence": "Calls the release a possible trigger and says the cause is not established." }, { "check": "recovery or stop criterion", "pass": true, "evidence": "Requires indicator recovery and stability over an observation window, with escalation otherwise." }, { "check": "no data deletion", "pass": true, "evidence": "Explicitly says not to delete production data." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 215, "word_limit_pass": true, "notes": "Good evidence preservation, reversible response, and authority caveat.", "task_id": "T1" }, { "blind_id": "T2-A", "objective_checks": [ { "check": "five questions", "pass": true, "evidence": "Contains exactly five numbered interview questions." }, { "check": "specific past behavior", "pass": true, "evidence": "Asks about the last real reservation attempt and subsequent actions." }, { "check": "20-minute allocation", "pass": true, "evidence": "Allocates 2+14+4 minutes per interview." }, { "check": "observation versus inference", "pass": true, "evidence": "Distinguishes recorded statements from interpretation and limits claims from five interviews." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 219, "word_limit_pass": true, "notes": "Neutral episode questions and a useful evidence table.", "task_id": "T2" }, { "blind_id": "T3-B", "objective_checks": [ { "check": "one decision owner", "pass": true, "evidence": "Defines one Approver with actual authority." }, { "check": "Friday deadline", "pass": true, "evidence": "Sets Friday as the decision deadline." }, { "check": "decision evidence", "pass": true, "evidence": "Calls for available funnel, support, usability, effort, risk, and assumption evidence." }, { "check": "consultation versus decision", "pass": true, "evidence": "Driver and Contributors advise; Approver decides without unanimity." } ], "objective_total": 4, "method_fit": 1, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [ "Allows the Approver to defer choosing an option on Friday if a fact is missing, despite the fixed choice requirement." ], "unsupported_claims": [], "word_count": 192, "word_limit_pass": true, "notes": "Sound role and evidence plan, but its fallback can miss the required Friday choice.", "task_id": "T3" }, { "blind_id": "T4-C", "objective_checks": [ { "check": "November 3 and 14:00", "pass": true, "evidence": "Includes November 3 at 14:00." }, { "check": "Civic Hall floor 2", "pass": true, "evidence": "Includes Civic Hall, floor 2." }, { "check": "free and capacity 20", "pass": true, "evidence": "States free and capacity 20." }, { "check": "October 31 and phone", "pass": true, "evidence": "Gives October 31 deadline and registration phone number." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 30, "word_limit_pass": true, "notes": "Clear logistics and call to register; no extra factual claims.", "task_id": "T4" }, { "blind_id": "T5-C", "objective_checks": [ { "check": "seven-day schedule", "pass": true, "evidence": "Lists Day 1 through Day 7." }, { "check": "20 minutes daily", "pass": true, "evidence": "Defines a 10+7+3 minute routine and 140 total." }, { "check": "closed-book recall", "pass": true, "evidence": "Requires written recall before looking at answers and final closed-book assessment." }, { "check": "check and correct errors", "pass": true, "evidence": "Checks source material, corrects errors, and logs gaps." } ], "objective_total": 4, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 185, "word_limit_pass": true, "notes": "Strong retrieval routine; day-specific focus can be read as adapting the general 20-concept routine.", "task_id": "T5" }, { "blind_id": "T6-C", "objective_checks": [ { "check": "read contribution rules", "pass": true, "evidence": "Directs reading contribution guide and project process." }, { "check": "search existing discussions", "pass": true, "evidence": "Directs checking issues, discussions, and recent changes." }, { "check": "reproduction or measurement", "pass": true, "evidence": "Calls for reproduction, timing method, and before/after comparison." }, { "check": "confirm scope before submission", "pass": false, "evidence": "Seeks maintainer guidance only if behavior must change; does not otherwise confirm scope with maintainers." } ], "objective_total": 3, "method_fit": 2, "actionability": 2, "evidence_boundaries": 2, "constraint_violations": [], "unsupported_claims": [], "word_count": 193, "word_limit_pass": true, "notes": "Preserves behavior and avoids fabricated speedup; scope confirmation is conditional.", "task_id": "T6" } ] }, "measured": { "mechanical": "answer count, frozen hashes, same-record leaf equality, whitespace word limit, input material token estimate", "semantic_model_assessment": "objective rubric criteria and all subjective quality scores are model assessed; not executed task results" }, "limitations": [ "Six English synthetic tasks and Korean public records; no randomized human users, repetitions or significance test.", "Manual record selection excludes retrieval errors and does not test network visualization or UI exporter.", "Single condition agent generates all six tasks; sequential context/task carryover may affect answers.", "One blind model judge; rubric ceiling and scoring bias possible. Objective semantic checks remain judgments, not external verification.", "No real incident recovery, interviews, learning retention, registration, OSS contribution or real-world outcome measured.", "Compact format is lossless nesting, not a validated task-specific reasoning system.", "No claims about the public contributor outcomes being independently verified.", "Actual provider token usage, reasoning tokens, cache, latency and billing unavailable." ], "korean_summary": { "headline": "6개 합성 과제에서는 지식 제공의 전반적 품질 우위를 확인하지 못했습니다.", "findings": [ "과제별 의미 기준은 무지원 24/24, 동일 자료 평면형 23/24, 동일 자료 중첩형 23/24였고, 모델 판단 점수는 모두 35/36이었습니다.", "중첩형은 동일 기록의 입력 토큰 추정량을 평면형보다 13.1% 줄였습니다. 실제 사용 토큰·비용 절감은 측정하지 못했습니다.", "220단어 제한 준수는 무지원 6/6, 평면형 3/6, 중첩형 6/6이었습니다. 소규모 단일 생성 결과이므로 일반화하지 않습니다.", "자료 제공 두 조건은 안내문에서 근거 없는 홍보 최상급을 제거했지만, 무지원은 원문 표현을 남겼습니다. 실무에서 사실과 과장을 나누는 예시이며 실제 사용자 효과를 검증한 것은 아닙니다.", "인터뷰와 의사결정 과제에서는 자료가 있어도 유도 질문이나 기한 미준수 위험이 남았습니다." ], "conclusion": "현재 근거는 출처와 한계가 보존되는 참고 자료 전달 및 입력 형식의 작은 효율 차이까지입니다. 그래프 사용, 실제 작업 성과, 인간 선호 또는 광범위한 답변 품질 개선을 입증하지 않습니다.", "grade_ambiguity": "T6의 범위 확인 기준에 일관되지 않은 채점 해석이 발견되었습니다. 원점수는 보존했으며, 엄격/완화 해석을 각각 일관되게 적용하면 세 조건이 모두 23/24 또는 24/24로 동률입니다. 원점수 1점 차이는 우위의 근거가 아닙니다." }, "publishable_claims": [ "Across six synthetic corpus-aligned single-run tasks, all conditions scored 35/36 on blind model judgment; no overall output-quality advantage established.", "Original criterion totals were 24/24 unaided and 23/24 each supplied-record condition. A task6 grading ambiguity makes that gap unreliable: all tie at 23 under strict reading or 24 under permissive reading.", "Identical full public records in nested JSON used 5959 versus 6857 o200k_base estimated input tokens, a 13.1 percent reduction versus flat leaf-path JSON; actual usage and cost unavailable.", "Word cap adherence was 6/6 unaided, 3/6 flat and 6/6 nested; small single-run observation only." ], "model_provenance": { "generation_requested_model": "gpt-6-luna", "review_requested_model": "gpt-6-sol", "review_evidence": "Actual collaboration.spawn_agent call used model=gpt-6-sol and fork_turns=none. Fresh Sol in messages was shorthand.", "backend_model_identity_independently_verified": false, "provider_usage": null }, "grade_adjudication": { "schema": "atlas-grade-adjudication-v1", "kind": "post-hoc unblinded evaluator audit; not independent blind regrading", "original_grades_preserved": true, "issue": { "task_id": "T6", "criterion": "confirm scope before submission", "frozen_wording_requires_maintainer_confirmation": false, "inconsistency": "Original grader accepted a conditional focused question plus scope checks in unaided, but required explicit maintainer confirmation in flat/compact. The frozen criterion does not specify unconditional maintainer confirmation.", "answer_evidence": { "unaided": "If the goal remains unclear, ask a focused question before coding; review diff before PR.", "flat_same_records": "If intended change is unclear, prepare a question; define narrow scope and prepare exact-scope summary before submission.", "compact_same_records": "Follow stated process; seek guidance if behavior must change; review diff for scope before submission." } }, "resolution": "Do not choose a new winner. Treat this one binary criterion as ambiguous; retain original scores and report sensitivity. No rubric changes or answer reruns.", "sensitivity": { "original_criterion_passes_model_reviewed": { "unaided": 24, "flat_same_records": 23, "compact_same_records": 23 }, "consistent_strict_explicit_maintainer_scope_confirmation": { "unaided": 23, "flat_same_records": 23, "compact_same_records": 23 }, "consistent_permissive_pre_submission_scope_review": { "unaided": 24, "flat_same_records": 24, "compact_same_records": 24 } }, "conclusion": "All conditions tie under both consistent readings. Original one-point criterion advantage for unaided is not robust. Subjective judgment remains 35/36 for every condition.", "independence_caveat": "The same evaluation coordinator inspected labels for this audit; no second blind human or model adjudicator was used." }, "criterion_label": "criterion_passes_model_reviewed: semantic task criteria graded by a model, not automatic objective tests; raw blind grades retain original objective_* field names as audit evidence.", "example_outputs": { "자료 없음": "Join us for \"The most innovative learning!\" on November 3 at 14:00 at Civic Hall, floor 2. Admission is free, and capacity is limited to 20 people. Registration closes October 31. To register, call 02-000-0000.", "동일 기록 평면형": "Free course - November 3\n\nJoin us November 3 at 14:00 at Civic Hall, floor 2. Admission is free; capacity is 20.\n\nRegister by October 31: call 02-000-0000.", "동일 기록 중첩형": "Course announcement\n\nJoin us for a learning course on November 3 at 14:00.\n\nCivic Hall, floor 2\nFree · Capacity: 20 people\nRegister by October 31.\n\nFor registration, call 02-000-0000." } }, "failure_transfer": { "schema": "openakashic-failure-transfer-combined-v1", "measured_at": "2026-10-28T05:56:48.201950+00:00", "primary": { "schema": "openakashic-failure-transfer-result-v1", "measured_at": "2026-10-28T05:51:34.163899+00:00", "protocol_sha256": "559004966a2ed004844a365d548608248d597e2a27f29dec67a66e4178e9b329", "freeze_sha256": "6101f94ecee8913e46bd095027495cea60bf93978615165ddc834e36f5094526", "frozen_at": "2026-10-28T05:47:58.132205+00:00", "design": { "schema": "openakashic-failure-transfer-v1", "created_at": "2026-10-28T05:47:57.956895+00:00", "requested_model": "gpt-6-luna", "arms": [ "method_only", "raw_experience", "linked_experience" ], "families": [ "backup", "join", "search" ], "generated_programs": 9, "fixtures_per_family": 6, "planned_case_executions": 54, "generation": "One fresh agent per arm, each receives 3 families. No other arms or test fixtures. Same tool-access restriction and one output per family. No post-score repairs. Within-arm context carryover possible.", "primary": "All 6 frozen fixtures pass per program, 3 programs per arm. Report per-family case counts, exceptions, and check failures; 54 executions are not 54 independent generations.", "hypotheses": [ "Does supplying concrete recorded risks improve objective implementation correctness over complete existing methods?", "Does regrouping the exact same reports by stored outcome_id improve correctness over original report arrays?" ], "strong_baseline": "Identical complete current contracts plus existing method cautions. No stripping warnings or withholding requirements to manufacture gain.", "source_scope": "Three deliberately selected public controlled examples; SQLite FK risk and search misses, join is a successful risk audit rather than a failed implementation.", "controls": [ "All records manually selected: no retrieval or graph navigation efficacy measured.", "Raw and linked experiences contain exactly identical full objects; linked regrouping only follows explicit outcome_id.", "No private experiences, credentials, actual bank data, or real customer fixtures.", "No inference of human utility, billing savings, generalization, statistical significance or model-family superiority.", "Requested model label available; actual backend identity and billed usage may be unavailable and remain null." ], "pre_generation_correction": "Draft harness Korean strings were found corrupted in transport; corrected before protocol freeze or candidate generation. Draft preserved privately in runtime, not an experimental output.", "docker_image": "sha256:5789aa194f757888f41fa3d48e14b6feaa1b57c0af0b56fcb1372d6474d3bff0", "actual_billed_usage": null, "actual_cost": null }, "summary": { "method_only": { "programs_fully_passed": 2, "programs": 3, "passed_cases": 17, "cases": 18, "families": { "backup": { "passed": 6, "cases": 6 }, "join": { "passed": 6, "cases": 6 }, "search": { "passed": 5, "cases": 6 } }, "input_characters": 10683, "input_utf8_bytes": 13530, "input_material_o200k_tokens": 3249, "actual_usage": null, "actual_cost": null, "failures": [ { "family": "search", "fixture": "search-1", "result": { "output": [], "unchanged": true, "actual_byte": null, "backup_observed": null, "input_unchanged": true, "returncode": 0, "stderr": "" } } ] }, "raw_experience": { "programs_fully_passed": 2, "programs": 3, "passed_cases": 17, "cases": 18, "families": { "backup": { "passed": 6, "cases": 6 }, "join": { "passed": 6, "cases": 6 }, "search": { "passed": 5, "cases": 6 } }, "input_characters": 20057, "input_utf8_bytes": 25395, "input_material_o200k_tokens": 6441, "actual_usage": null, "actual_cost": null, "failures": [ { "family": "search", "fixture": "search-1", "result": { "output": [], "unchanged": true, "actual_byte": null, "backup_observed": null, "input_unchanged": true, "returncode": 0, "stderr": "" } } ] }, "linked_experience": { "programs_fully_passed": 3, "programs": 3, "passed_cases": 18, "cases": 18, "families": { "backup": { "passed": 6, "cases": 6 }, "join": { "passed": 6, "cases": 6 }, "search": { "passed": 6, "cases": 6 } }, "input_characters": 20727, "input_utf8_bytes": 26065, "input_material_o200k_tokens": 6503, "actual_usage": null, "actual_cost": null, "failures": [] } }, "comparison": "Manually selected existing public method versus added real stored synthetic-example reports versus exact same reports regrouped by outcome_id. No independent retrieval comparison.", "limits": [ "Three selected families, one implementation per arm and family. Six fixtures per program are clustered checks, not independent generations.", "All arms already receive complete task contracts and baseline method caveats; ceiling effects possible.", "Prior join case is successful risk detection, not an observed failed join. Backups expose FK inconsistency; search prior cases include actual misses.", "No private or real customer data; examples are synthetic. No measured user benefit, reviewer time, actual billing or statistical superiority.", "One agent generates three programs within each arm; within-arm context carryover possible. Requested model label is not independently verified backend identity.", "Exact same experience objects and no additional claims in linked arm. This is packaging, not graph reasoning." ] }, "followup": { "schema": "openakashic-failure-transfer-result-v1", "measured_at": "2026-10-28T05:55:38.141407+00:00", "protocol_sha256": "1bdfae6d668f038396a938d495b42dd10012c6bc9ab8fa96fdb8f7bc42f15fc1", "freeze_sha256": "b69266a505eb80ad2655bc0acd6f7e93a433d25d0f1fcd3d6da99135b7931ed1", "frozen_at": "2026-10-28T05:52:08.199235+00:00", "design": { "schema": "openakashic-failure-transfer-transport-followup-v1", "created_at": "2026-10-28T05:52:08.195398+00:00", "requested_model": "gpt-6-luna", "arms": [ "method_only", "raw_experience", "linked_experience" ], "families": [ "backup", "join", "search" ], "generated_programs": 9, "fixtures_per_family": 6, "planned_case_executions": 54, "generation": "Fresh agent per arm; no primary outputs or grades disclosed. Same reference information and task contracts. Write source with PowerShell WriteAllText and UTF8 encoding directly to WSL UNC, never native-pipe source text. One implementation per family. No test access or execution.", "primary": "All 6 frozen fixtures pass per program, 3 programs per arm. Report per-family case counts, exceptions, and check failures; 54 executions are not 54 independent generations.", "hypotheses": [ "Does supplying concrete recorded risks improve objective implementation correctness over complete existing methods?", "Does regrouping the exact same reports by stored outcome_id improve correctness over original report arrays?" ], "strong_baseline": "Identical complete current contracts plus existing method cautions. No stripping warnings or withholding requirements to manufacture gain.", "source_scope": "Three deliberately selected public controlled examples; SQLite FK risk and search misses, join is a successful risk audit rather than a failed implementation.", "controls": [ "All records manually selected: no retrieval or graph navigation efficacy measured.", "Raw and linked experiences contain exactly identical full objects; linked regrouping only follows explicit outcome_id.", "No private experiences, credentials, actual bank data, or real customer fixtures.", "No inference of human utility, billing savings, generalization, statistical significance or model-family superiority.", "Requested model label available; actual backend identity and billed usage may be unavailable and remain null." ], "pre_generation_correction": "Draft harness Korean strings were found corrupted in transport; corrected before protocol freeze or candidate generation. Draft preserved privately in runtime, not an experimental output.", "docker_image": "sha256:5789aa194f757888f41fa3d48e14b6feaa1b57c0af0b56fcb1372d6474d3bff0", "actual_billed_usage": null, "actual_cost": null, "original_freeze_sha256": "6101f94ecee8913e46bd095027495cea60bf93978615165ddc834e36f5094526", "followup_reason": "Primary 17/18,17/18,18/18 differs only on Korean particle case; observed source files show method_only double-escaped Unicode and raw_experience question-mark suffix corruption. Preserve original outputs; regenerate ALL arms fresh with one shared reliable file transport. No per-output repairs. This uses previously evaluated fixtures, not a new holdout." }, "summary": { "method_only": { "programs_fully_passed": 3, "programs": 3, "passed_cases": 18, "cases": 18, "families": { "backup": { "passed": 6, "cases": 6 }, "join": { "passed": 6, "cases": 6 }, "search": { "passed": 6, "cases": 6 } }, "input_characters": 11140, "input_utf8_bytes": 13987, "input_material_o200k_tokens": 3345, "actual_usage": null, "actual_cost": null, "failures": [] }, "raw_experience": { "programs_fully_passed": 2, "programs": 3, "passed_cases": 17, "cases": 18, "families": { "backup": { "passed": 6, "cases": 6 }, "join": { "passed": 5, "cases": 6 }, "search": { "passed": 6, "cases": 6 } }, "input_characters": 20514, "input_utf8_bytes": 25852, "input_material_o200k_tokens": 6537, "actual_usage": null, "actual_cost": null, "failures": [ { "family": "join", "fixture": "join-1", "result": { "output": { "left_blank": 2, "right_blank": 1, "left_duplicate_rows": 2, "right_duplicate_rows": 2, "matched_left_rows": 2, "matched_right_rows": 2, "unmatched_left_rows": 0, "unmatched_right_rows": 0, "inner_rows": 4, "left_join_rows": 4, "full_join_rows": 4, "actual_cardinality": "many_to_many", "expected_satisfied": false }, "unchanged": true, "actual_byte": null, "backup_observed": null, "input_unchanged": true, "returncode": 0, "stderr": "" } } ] }, "linked_experience": { "programs_fully_passed": 3, "programs": 3, "passed_cases": 18, "cases": 18, "families": { "backup": { "passed": 6, "cases": 6 }, "join": { "passed": 6, "cases": 6 }, "search": { "passed": 6, "cases": 6 } }, "input_characters": 21184, "input_utf8_bytes": 26522, "input_material_o200k_tokens": 6599, "actual_usage": null, "actual_cost": null, "failures": [] } }, "comparison": "Manually selected existing public method versus added real stored synthetic-example reports versus exact same reports regrouped by outcome_id. No independent retrieval comparison.", "limits": [ "Three selected families, one implementation per arm and family. Six fixtures per program are clustered checks, not independent generations.", "All arms already receive complete task contracts and baseline method caveats; ceiling effects possible.", "Prior join case is successful risk detection, not an observed failed join. Backups expose FK inconsistency; search prior cases include actual misses.", "No private or real customer data; examples are synthetic. No measured user benefit, reviewer time, actual billing or statistical superiority.", "One agent generates three programs within each arm; within-arm context carryover possible. Requested model label is not independently verified backend identity.", "Exact same experience objects and no additional claims in linked arm. This is packaging, not graph reasoning." ] }, "conclusion": "후속에서 방법만 제공한 조건과 연결형 경험 조건이 모두 18/18로 같았습니다. 원래 경험 기록 조건은 빈 키를 미매칭 행 수에서 누락해 17/18이었습니다. 경험 추가나 그래프 연결이 구현 품질을 높였다는 근거는 확인되지 않았습니다. 작은 합성 과제의 단회 관측이며 일반적인 유해성도 입증하지 않습니다.", "audit": { "original_counts": [ 17, 17, 18 ], "denominator": 18, "only_failed_case": "search-1", "method_only": "source regex and suffix values double escaped", "raw_experience": "source suffix strings replaced by question marks", "linked_experience": "source has valid escapes", "interpretation": "Original gap is confounded by artifact production and cannot establish experience-packaging benefit. All outputs and scores retained.", "followup_freeze": "b69266a505eb80ad2655bc0acd6f7e93a433d25d0f1fcd3d6da99135b7931ed1", "followup_new_holdout": false }, "followup_failure": "raw_experience join-1: blank keys counted as blanks but omitted from unmatched rows, left join rows and full join rows. Expected unmatched 2/1, left rows6, full rows7; returned0/0,4,4. Do not infer causation from a single generated program." } }