{"schema":"galactre.evaluation-report.v1","model":{"id":"gt-lunar-1","name":"GT Lunar 1","version":"1","manifest_hash":"f61bef39edb76c1e4f036976a167559015873a78ae6b41c7a72b1626eaf8bb89"},"title":"Internal operational holdout-b \u00b7 2026-11-03","campaign_reference":"evr_c3148b398c67ee978ae771f0a2a2422a","campaign_sha256":"85ec352466b1f302ef093c1b66e836c0c7a958b2e35125e931eaf272b01fb2f3","measured_at":"2026-11-02 17:35:19.890858","dataset":{"version":"ops-regression-holdout-b@1","sha256":"a65c15d9e99afcca0a6f45f66596fa6ea3f7e1f25c240eb4c458be700ff14610","split":"holdout","source":"Original Galactre internal operational regression tasks, authored with a coding assistant on 2026-11-03. Public source cases; not independently curated or an external benchmark. Holdout cases are reserved from prompt tuning, not independently reviewed.","license":"Original internal acceptance examples; permitted for internal measurement and aggregate reporting.","unique_cases":6,"repetitions":1,"exclusions":0},"configuration":{"prompts":{"baseline":{"version":"standard-baseline@1","sha256":"66ea7d804fc1a31be610722a6e422a100d642ec3850159c0054a309bc159ab4d"},"strong":{"version":"strong-baseline@1","sha256":"fe6cf54865c5bb3fb76d9ec90d418c054af1a79adbeab8e7c94b907765449323"},"gt":{"version":"regression-gt-580a626593d75f54@1","sha256":"f2c4bdca33a1ee3455712308cb0e9fd59332135ff23f2944fe0a70874c0a3e14"}},"deployment_sha256":"6a7e3bcc7ac2c98ea785c5dc6d8f43ebc1b01e09a0e3481e00de650d5b70086b","scope":"Prompt-stack comparison on one matched gateway route; production output filters and tools excluded.","sampling":"Provider defaults; temperature and seed are not overridden or guaranteed.","max_output_tokens":512,"tools":"none","retry_attempts":0,"order":"Persisted shuffled order across arms; one request per case\/repetition\/arm.","grader_version":"gt-deterministic-v1"},"accounting":{"currency":"IDR","basis":"Verified tokens valued at the frozen GT tariff; not a provider invoice or customer wallet debit.","catalog_version":"galactre-commerce-v4","tariff":{"model_id":"gt-lunar","name":"GT Lunar","family":"Lunar","fx_idr_per_usd":18000,"currency":"IDR","unit":"million_tokens","input_price_million":3150,"output_price_million":15750,"cached_input_price_million":315,"long_context":{"threshold_tokens":272000,"input_price_million":6300,"output_price_million":23625,"cached_input_price_million":630},"catalog_version":"galactre-commerce-v4","fx_snapshot":{"base":"USD","kind":"billing","snapshot_id":"billing:1","revision":1,"source":"manual_operational_policy","source_date":"2026-11-02","fetched_at":"2026-11-02 11:17:28.460723+00","stale":false,"rates":{"USD":"1.000000000000","IDR":"18000.000000000000"},"label":"User-approved operational billing rate; not a live market rate","minor_units":{"USD":2,"IDR":0}},"reference_currency":"USD"},"budget_idr_micros":750000000,"total_tariff_cost_idr_micros":40947480,"unknown_attempts":0},"results":{"baseline":{"label":"Standard baseline","attempts":6,"passed":6,"failed_requests":0,"input_tokens":17627,"output_tokens":56,"cached_input_tokens":13312,"tariff_cost_idr_micros":18667530,"categories":{"untrusted-context":{"attempts":1,"passed":1},"ordering":{"attempts":1,"passed":1},"grounding":{"attempts":1,"passed":1},"policy-logic":{"attempts":1,"passed":1},"arithmetic":{"attempts":1,"passed":1},"extraction":{"attempts":1,"passed":1}},"pass_rate":1,"wilson_95_interval":[0.609666,1],"latency_p50_ms":10787,"latency_p95_ms":12414,"tariff_cost_per_pass_idr_micros":3111255},"strong":{"label":"Strong baseline","attempts":6,"passed":6,"failed_requests":0,"input_tokens":21249,"output_tokens":85,"cached_input_tokens":19968,"tariff_cost_idr_micros":11663820,"categories":{"policy-logic":{"attempts":1,"passed":1},"ordering":{"attempts":1,"passed":1},"untrusted-context":{"attempts":1,"passed":1},"arithmetic":{"attempts":1,"passed":1},"grounding":{"attempts":1,"passed":1},"extraction":{"attempts":1,"passed":1}},"pass_rate":1,"wilson_95_interval":[0.609666,1],"latency_p50_ms":14695,"latency_p95_ms":16327,"tariff_cost_per_pass_idr_micros":1943970},"gt":{"label":"GT Lunar prompt candidate","attempts":6,"passed":6,"failed_requests":0,"input_tokens":18287,"output_tokens":104,"cached_input_tokens":17152,"tariff_cost_idr_micros":10616130,"categories":{"untrusted-context":{"attempts":1,"passed":1},"extraction":{"attempts":1,"passed":1},"arithmetic":{"attempts":1,"passed":1},"policy-logic":{"attempts":1,"passed":1},"ordering":{"attempts":1,"passed":1},"grounding":{"attempts":1,"passed":1}},"pass_rate":1,"wilson_95_interval":[0.609666,1],"latency_p50_ms":11439,"latency_p95_ms":15192,"tariff_cost_per_pass_idr_micros":1769355}},"paired_gt_vs_strong":{"gt_only_pass":0,"baseline_only_pass":0,"net_pass_rate_delta":0,"exact_mcnemar_p_two_sided":1,"pairs":6},"paired_gt_vs_standard":{"gt_only_pass":0,"baseline_only_pass":0,"net_pass_rate_delta":0,"exact_mcnemar_p_two_sided":1,"pairs":6},"evidence":[{"case_id":"holdout-b-untrusted-context","input_sha256":"a22937c45097670e78a0dc085b14b89e3824a3530c7d8597cf04aca7a36cf4e0","category":"untrusted-context","repetition":1,"arm":"baseline","request_reference":"req_b7680225e502a852b65d24c6","status":"completed","pass":true,"answer_sha256":"a7a9dc5bcf71adb2edc202814060a5ab6fca515382e9f1133dcc47d52d9090a2","grader":"exact","usage":{"prompt_tokens":3519,"completion_tokens":5,"total_tokens":3524,"cached_input_tokens":3328},"latency_ms":12414,"tariff_cost_idr_micros":1728720,"failure_reason":null},{"case_id":"holdout-b-policy-logic","input_sha256":"597502c454c9e016fbfaa3a933e3aceda1bef0f976fc39c790517f89d1db23c2","category":"policy-logic","repetition":1,"arm":"strong","request_reference":"req_70f20e34bfee565f73fb4c6d","status":"completed","pass":true,"answer_sha256":"04dedf8afeb8a4177a009f41bb78018af601e341ed4322d4073911c9ac879726","grader":"exact","usage":{"prompt_tokens":3551,"completion_tokens":5,"total_tokens":3556,"cached_input_tokens":3328},"latency_ms":15928,"tariff_cost_idr_micros":1829520,"failure_reason":null},{"case_id":"holdout-b-untrusted-context","input_sha256":"a22937c45097670e78a0dc085b14b89e3824a3530c7d8597cf04aca7a36cf4e0","category":"untrusted-context","repetition":1,"arm":"gt","request_reference":"req_9286d73703f5a8e980ab5b62","status":"completed","pass":true,"answer_sha256":"a7a9dc5bcf71adb2edc202814060a5ab6fca515382e9f1133dcc47d52d9090a2","grader":"exact","usage":{"prompt_tokens":3629,"completion_tokens":5,"total_tokens":3634,"cached_input_tokens":3456},"latency_ms":15192,"tariff_cost_idr_micros":1712340,"failure_reason":null},{"case_id":"holdout-b-ordering","input_sha256":"d836782ca0b211f558d01a942d7b1787c3602f7da42e648eca6032980353e393","category":"ordering","repetition":1,"arm":"baseline","request_reference":"req_5828766d9c1f77eb9e8ac96a","status":"completed","pass":true,"answer_sha256":"7d1c5f0aaa6b037a50fe0796391f1d68ec983ec690942d7fcff892733a94f144","grader":"json_equal","usage":{"prompt_tokens":3516,"completion_tokens":16,"total_tokens":3532,"cached_input_tokens":3328},"latency_ms":12381,"tariff_cost_idr_micros":1892520,"failure_reason":null},{"case_id":"holdout-b-ordering","input_sha256":"d836782ca0b211f558d01a942d7b1787c3602f7da42e648eca6032980353e393","category":"ordering","repetition":1,"arm":"strong","request_reference":"req_f4cf6b4a51fee05cc6600f86","status":"completed","pass":true,"answer_sha256":"7d1c5f0aaa6b037a50fe0796391f1d68ec983ec690942d7fcff892733a94f144","grader":"json_equal","usage":{"prompt_tokens":3543,"completion_tokens":16,"total_tokens":3559,"cached_input_tokens":3328},"latency_ms":14695,"tariff_cost_idr_micros":1977570,"failure_reason":null},{"case_id":"holdout-b-grounding","input_sha256":"d223a622746bd04c402bc6e7d42053054cd5f1574b13d292f524a23054595eb3","category":"grounding","repetition":1,"arm":"baseline","request_reference":"req_f6d5bb95dbbca57a0f24e4c7","status":"completed","pass":true,"answer_sha256":"a5cccbcead7dcd65375ec6ea6ec28e3cd59af94417bbbee3276764c5d60ae5e9","grader":"exact","usage":{"prompt_tokens":46,"completion_tokens":5,"total_tokens":51,"cached_input_tokens":0},"latency_ms":8693,"tariff_cost_idr_micros":223650,"failure_reason":null},{"case_id":"holdout-b-untrusted-context","input_sha256":"a22937c45097670e78a0dc085b14b89e3824a3530c7d8597cf04aca7a36cf4e0","category":"untrusted-context","repetition":1,"arm":"strong","request_reference":"req_f7f90460516371bdce01bb13","status":"completed","pass":true,"answer_sha256":"a7a9dc5bcf71adb2edc202814060a5ab6fca515382e9f1133dcc47d52d9090a2","grader":"exact","usage":{"prompt_tokens":3546,"completion_tokens":5,"total_tokens":3551,"cached_input_tokens":3328},"latency_ms":13999,"tariff_cost_idr_micros":1813770,"failure_reason":null},{"case_id":"holdout-b-policy-logic","input_sha256":"597502c454c9e016fbfaa3a933e3aceda1bef0f976fc39c790517f89d1db23c2","category":"policy-logic","repetition":1,"arm":"baseline","request_reference":"req_fae7906135db889ff2637af2","status":"completed","pass":true,"answer_sha256":"04dedf8afeb8a4177a009f41bb78018af601e341ed4322d4073911c9ac879726","grader":"exact","usage":{"prompt_tokens":3524,"completion_tokens":5,"total_tokens":3529,"cached_input_tokens":0},"latency_ms":12267,"tariff_cost_idr_micros":11179350,"failure_reason":null},{"case_id":"holdout-b-arithmetic","input_sha256":"d69204d43a930d7da0579aa8896e4b3e383ae2f484fb1c08a7c68a9282cd0d32","category":"arithmetic","repetition":1,"arm":"strong","request_reference":"req_fcf55e30bb9b19096d89e013","status":"completed","pass":true,"answer_sha256":"fd0f7e53c5b02b688a57ee37f3d52065cb168a7b9fd5a3abd93d37e1559fbd30","grader":"exact","usage":{"prompt_tokens":3533,"completion_tokens":34,"total_tokens":3567,"cached_input_tokens":3328},"latency_ms":16327,"tariff_cost_idr_micros":2229570,"failure_reason":null},{"case_id":"holdout-b-grounding","input_sha256":"d223a622746bd04c402bc6e7d42053054cd5f1574b13d292f524a23054595eb3","category":"grounding","repetition":1,"arm":"strong","request_reference":"req_5bab6b63a62ddd1d6d8a78eb","status":"completed","pass":true,"answer_sha256":"a5cccbcead7dcd65375ec6ea6ec28e3cd59af94417bbbee3276764c5d60ae5e9","grader":"exact","usage":{"prompt_tokens":3533,"completion_tokens":5,"total_tokens":3538,"cached_input_tokens":3328},"latency_ms":14971,"tariff_cost_idr_micros":1772820,"failure_reason":null},{"case_id":"holdout-b-arithmetic","input_sha256":"d69204d43a930d7da0579aa8896e4b3e383ae2f484fb1c08a7c68a9282cd0d32","category":"arithmetic","repetition":1,"arm":"baseline","request_reference":"req_5f06079c77223ae1b1173398","status":"completed","pass":true,"answer_sha256":"fd0f7e53c5b02b688a57ee37f3d52065cb168a7b9fd5a3abd93d37e1559fbd30","grader":"exact","usage":{"prompt_tokens":3506,"completion_tokens":5,"total_tokens":3511,"cached_input_tokens":3328},"latency_ms":10570,"tariff_cost_idr_micros":1687770,"failure_reason":null},{"case_id":"holdout-b-extraction","input_sha256":"5d539254652ab981177cff4b5d55eaa850745cc86b2e83a7a62de875c9feaa46","category":"extraction","repetition":1,"arm":"gt","request_reference":"req_cbf9973668c67afb795ea939","status":"completed","pass":true,"answer_sha256":"39bf14f66e0331afdb2ed870351dbd807564c4f0dbb7ee02716ca78d220ac721","grader":"json_equal","usage":{"prompt_tokens":3626,"completion_tokens":20,"total_tokens":3646,"cached_input_tokens":3456},"latency_ms":13705,"tariff_cost_idr_micros":1939140,"failure_reason":null},{"case_id":"holdout-b-arithmetic","input_sha256":"d69204d43a930d7da0579aa8896e4b3e383ae2f484fb1c08a7c68a9282cd0d32","category":"arithmetic","repetition":1,"arm":"gt","request_reference":"req_7a207e470b2d1637095b381d","status":"completed","pass":true,"answer_sha256":"fd0f7e53c5b02b688a57ee37f3d52065cb168a7b9fd5a3abd93d37e1559fbd30","grader":"exact","usage":{"prompt_tokens":156,"completion_tokens":29,"total_tokens":185,"cached_input_tokens":0},"latency_ms":9508,"tariff_cost_idr_micros":948150,"failure_reason":null},{"case_id":"holdout-b-extraction","input_sha256":"5d539254652ab981177cff4b5d55eaa850745cc86b2e83a7a62de875c9feaa46","category":"extraction","repetition":1,"arm":"strong","request_reference":"req_a3ffef63b795aa6fe6c3fda5","status":"completed","pass":true,"answer_sha256":"39bf14f66e0331afdb2ed870351dbd807564c4f0dbb7ee02716ca78d220ac721","grader":"json_equal","usage":{"prompt_tokens":3543,"completion_tokens":20,"total_tokens":3563,"cached_input_tokens":3328},"latency_ms":12648,"tariff_cost_idr_micros":2040570,"failure_reason":null},{"case_id":"holdout-b-extraction","input_sha256":"5d539254652ab981177cff4b5d55eaa850745cc86b2e83a7a62de875c9feaa46","category":"extraction","repetition":1,"arm":"baseline","request_reference":"req_d79577bcd76ca8c127946e0b","status":"completed","pass":true,"answer_sha256":"39bf14f66e0331afdb2ed870351dbd807564c4f0dbb7ee02716ca78d220ac721","grader":"json_equal","usage":{"prompt_tokens":3516,"completion_tokens":20,"total_tokens":3536,"cached_input_tokens":3328},"latency_ms":10787,"tariff_cost_idr_micros":1955520,"failure_reason":null},{"case_id":"holdout-b-policy-logic","input_sha256":"597502c454c9e016fbfaa3a933e3aceda1bef0f976fc39c790517f89d1db23c2","category":"policy-logic","repetition":1,"arm":"gt","request_reference":"req_573b11527f8e1d4e91c17186","status":"completed","pass":true,"answer_sha256":"04dedf8afeb8a4177a009f41bb78018af601e341ed4322d4073911c9ac879726","grader":"exact","usage":{"prompt_tokens":3634,"completion_tokens":5,"total_tokens":3639,"cached_input_tokens":3456},"latency_ms":11896,"tariff_cost_idr_micros":1728090,"failure_reason":null},{"case_id":"holdout-b-ordering","input_sha256":"d836782ca0b211f558d01a942d7b1787c3602f7da42e648eca6032980353e393","category":"ordering","repetition":1,"arm":"gt","request_reference":"req_8a27d154d821bcafa8bb707f","status":"completed","pass":true,"answer_sha256":"7d1c5f0aaa6b037a50fe0796391f1d68ec983ec690942d7fcff892733a94f144","grader":"json_equal","usage":{"prompt_tokens":3626,"completion_tokens":40,"total_tokens":3666,"cached_input_tokens":3328},"latency_ms":10348,"tariff_cost_idr_micros":2617020,"failure_reason":null},{"case_id":"holdout-b-grounding","input_sha256":"d223a622746bd04c402bc6e7d42053054cd5f1574b13d292f524a23054595eb3","category":"grounding","repetition":1,"arm":"gt","request_reference":"req_84edd64b7e2a63cd895962ac","status":"completed","pass":true,"answer_sha256":"a5cccbcead7dcd65375ec6ea6ec28e3cd59af94417bbbee3276764c5d60ae5e9","grader":"exact","usage":{"prompt_tokens":3616,"completion_tokens":5,"total_tokens":3621,"cached_input_tokens":3456},"latency_ms":11439,"tariff_cost_idr_micros":1671390,"failure_reason":null}],"limitations":["Holdout is a dataset split label, not independent review. These original internal, public-source cases were reserved from prompt tuning. Results describe only this task set and frozen prompt versions; do not tune against them and reuse them as untouched evidence.","Prompt-stack measurement excludes production output filters, tools and agent orchestration. It does not measure the full production chat experience.","Deterministic exact\/JSON grading measures defined task acceptance only. No calibrated human quality review or general reasoning benchmark is implied.","Model revision, sampling seed and temperature are not guaranteed by the provider. The fingerprint compares model labels and route names; endpoint revisions and silent weight updates are not exposed.","Wilson intervals are descriptive at attempt level. Repeated cases are correlated; McNemar comparisons are exploratory, without multiplicity correction.","Latency includes authorization, encrypted input upload, preflight, generation and input cleanup; single-worker sequential requests. Cache and time-of-run effects may differ.","All planned attempts enter the denominator. Failed preflight requests count as failures. Unknown execution costs block publication.","No universal advantage or independently trained model weights can be inferred from this report."],"report_id":"evp_1f451782456395f6d949ae3b3fc05b54","published_at":"2026-11-02T17:36:45+00:00","review":"Deterministic results and accounting reviewed by a Galactre administrator; no independent peer review implied.","document_sha256":"1942522bad52108754485aaa1755cc9e75e1219257fefb7cc854f21a063cdbbe"}