{
  "seed": 20261012,
  "bootstrap_repetitions": 100000,
  "method": "Paired task percentile bootstrap: resample 50 whole task rows, retaining all four trials and all runs together. Historical Responses run is descriptive only. Intervals describe task sampling under stable-run assumptions; they do not identify global run effects or give fixed-benchmark future-run prediction bounds.",
  "runs": {
    "native_v1": {
      "path": "data/simulations/airline_native_azure_gpt52_high_baseline_v1/results.json",
      "sha256": "436e45285cccd73642585252a7dbe8f7394ffc45970afbb03be1597cb513ec1e",
      "source_commit": "cd336f5ea26658ade944d8af805de6f74582e01b",
      "successes": 166,
      "pass": [
        83.0,
        75.66666666666666,
        69.5,
        64.0
      ],
      "count_distribution": {
        "0": 4,
        "1": 1,
        "2": 2,
        "3": 11,
        "4": 32
      },
      "trial_successes": {
        "0": 41.0,
        "1": 43.0,
        "2": 39.0,
        "3": 43.0
      },
      "trial_seeds": [
        1567,
        361454,
        373753,
        626729
      ],
      "termination": {
        "user_stop": 200
      },
      "tool_calls": 2101,
      "tool_errors": 2,
      "agent_cost": 20.0522329,
      "user_cost": 2.96778055,
      "agent_usage": {},
      "agent_info": {
        "implementation": "llm_agent",
        "llm": "azure/gpt-5.2",
        "llm_args": {
          "reasoning_effort": "high"
        },
        "voice_settings": null
      },
      "max_steps": 200,
      "max_errors": 10,
      "simulation_seed": 300
    },
    "native_v2": {
      "path": "data/simulations/airline_native_azure_gpt52_high_baseline_v2/results.json",
      "sha256": "bb45befbc34d2466026e562a8c2e07ef334c3ffaf89238cbe65b986f1a4ffc01",
      "source_commit": "cd336f5ea26658ade944d8af805de6f74582e01b",
      "successes": 160,
      "pass": [
        80.0,
        70.66666666666666,
        63.5,
        57.99999999999999
      ],
      "count_distribution": {
        "0": 4,
        "1": 1,
        "2": 5,
        "3": 11,
        "4": 29
      },
      "trial_successes": {
        "0": 43.0,
        "1": 42.0,
        "2": 40.0,
        "3": 35.0
      },
      "trial_seeds": [
        1567,
        361454,
        373753,
        626729
      ],
      "termination": {
        "user_stop": 200
      },
      "tool_calls": 2171,
      "tool_errors": 4,
      "agent_cost": 19.4529083,
      "user_cost": 2.85183675,
      "agent_usage": {},
      "agent_info": {
        "implementation": "llm_agent",
        "llm": "azure/gpt-5.2",
        "llm_args": {
          "reasoning_effort": "high"
        },
        "voice_settings": null
      },
      "max_steps": 200,
      "max_errors": 10,
      "simulation_seed": 300
    },
    "responses": {
      "path": "data/simulations/airline_adapter_azure_gpt52_high_baseline_v1/results.json",
      "sha256": "4f3ace34c1b524189fd0c0c449be3d93575f26bd70b49202333fb545d459584d",
      "source_commit": "abadce934efd0fa9e6a28c443e3182e54a793b23",
      "successes": 164,
      "pass": [
        82.0,
        74.33333333333334,
        69.0,
        66.0
      ],
      "count_distribution": {
        "0": 4,
        "2": 7,
        "3": 6,
        "4": 33
      },
      "trial_successes": {
        "0": 39.0,
        "1": 40.0,
        "2": 41.0,
        "3": 44.0
      },
      "trial_seeds": [
        1567,
        361454,
        373753,
        626729
      ],
      "termination": {
        "user_stop": 200
      },
      "tool_calls": 1683,
      "tool_errors": 4,
      "agent_cost": 15.8764361,
      "user_cost": 2.9167586,
      "agent_usage": {},
      "agent_info": {
        "implementation": "platform_adapter",
        "llm": "gpt-5.2",
        "llm_args": {
          "reasoning_effort": "high",
          "pricing_model": "gpt-5.2"
        },
        "voice_settings": null
      },
      "max_steps": 200,
      "max_errors": 10,
      "simulation_seed": 300
    },
    "mcp_chat": {
      "path": "data/simulations/airline_adapter_chat_gpt52_high_baseline_v1/results.json",
      "sha256": "e2786602319924c5fc76d74f8e293ba5ae1170a03ff44ee3be1e300c7629fe08",
      "source_commit": "9395f8c1dccb7f8e3524a592e8f6b341ba464e9a",
      "successes": 156,
      "pass": [
        78.0,
        67.66666666666669,
        61.0,
        56.00000000000001
      ],
      "count_distribution": {
        "0": 3,
        "1": 4,
        "2": 5,
        "3": 10,
        "4": 28
      },
      "trial_successes": {
        "0": 37.0,
        "1": 39.0,
        "2": 40.0,
        "3": 40.0
      },
      "trial_seeds": [
        1567,
        361454,
        373753,
        626729
      ],
      "termination": {
        "user_stop": 200
      },
      "tool_calls": 1984,
      "tool_errors": 2,
      "agent_cost": 18.13175525,
      "user_cost": 2.99824735,
      "agent_usage": {},
      "agent_info": {
        "implementation": "platform_adapter",
        "llm": "azure/gpt-5.2",
        "llm_args": {
          "reasoning_effort": "high"
        },
        "voice_settings": null
      },
      "max_steps": 200,
      "max_errors": 10,
      "simulation_seed": 300
    }
  },
  "comparisons": {
    "native_v2_minus_native_v1": {
      "difference_pp": [
        -3.0,
        -5.0,
        -6.0,
        -6.0
      ],
      "task_bootstrap_ci95_pp": [
        [
          -7.5,
          1.5
        ],
        [
          -13.0,
          2.999999999999999
        ],
        [
          -17.0,
          5.0
        ],
        [
          -20.0,
          8.0
        ]
      ],
      "task_bootstrap_ci90_pp": [
        [
          -7.000000000000001,
          1.0
        ],
        [
          -11.666666666666666,
          1.6666666666666663
        ],
        [
          -15.5,
          3.0
        ],
        [
          -18.0,
          6.0
        ]
      ],
      "task_standard_error_pp": [
        2.32992949004287,
        4.0476190476190474,
        5.64963426665291,
        7.233707943526949
      ],
      "tasks_better": [
        "1",
        "9",
        "32",
        "41",
        "42",
        "45",
        "48"
      ],
      "tasks_worse": [
        "6",
        "12",
        "15",
        "17",
        "20",
        "24",
        "25",
        "29",
        "33",
        "37",
        "43",
        "49"
      ],
      "tasks_tied": 31,
      "perfect_task_discordance": {
        "gains": 5,
        "losses": 8,
        "both_perfect": 24,
        "exact_mcnemar_two_sided_p": 0.5810546875
      },
      "conversation_discordance": {
        "gains": 13,
        "losses": 19,
        "changed_outcomes": 32
      }
    },
    "mcp_chat_minus_native_v1": {
      "difference_pp": [
        -5.0,
        -8.0,
        -8.5,
        -8.0
      ],
      "task_bootstrap_ci95_pp": [
        [
          -9.5,
          -0.5
        ],
        [
          -15.666666666666664,
          -0.33333333333333326
        ],
        [
          -19.5,
          2.5
        ],
        [
          -22.0,
          6.0
        ]
      ],
      "task_bootstrap_ci90_pp": [
        [
          -9.0,
          -1.0
        ],
        [
          -14.333333333333334,
          -1.333333333333334
        ],
        [
          -17.5,
          0.5
        ],
        [
          -20.0,
          4.0
        ]
      ],
      "task_standard_error_pp": [
        2.3690177073967136,
        3.990919398048946,
        5.651891502056671,
        7.472398188926867
      ],
      "tasks_better": [
        "1",
        "9",
        "31",
        "39",
        "41",
        "48"
      ],
      "tasks_worse": [
        "3",
        "6",
        "14",
        "15",
        "17",
        "18",
        "19",
        "20",
        "23",
        "24",
        "25",
        "30",
        "33",
        "43",
        "45"
      ],
      "tasks_tied": 29,
      "perfect_task_discordance": {
        "gains": 5,
        "losses": 9,
        "both_perfect": 23,
        "exact_mcnemar_two_sided_p": 0.4239501953125
      },
      "conversation_discordance": {
        "gains": 11,
        "losses": 21,
        "changed_outcomes": 32
      }
    },
    "mcp_chat_minus_native_v2": {
      "difference_pp": [
        -2.0,
        -3.0,
        -2.5,
        -2.0
      ],
      "task_bootstrap_ci95_pp": [
        [
          -6.5,
          2.5
        ],
        [
          -10.0,
          3.6666666666666665
        ],
        [
          -12.0,
          6.5
        ],
        [
          -14.000000000000002,
          10.0
        ]
      ],
      "task_bootstrap_ci90_pp": [
        [
          -5.5,
          1.5
        ],
        [
          -8.666666666666668,
          2.666666666666667
        ],
        [
          -10.5,
          5.5
        ],
        [
          -12.0,
          8.0
        ]
      ],
      "task_standard_error_pp": [
        2.240626734479731,
        3.5215565513303475,
        4.804865731097753,
        6.05417717154774
      ],
      "tasks_better": [
        "12",
        "15",
        "29",
        "31",
        "37",
        "39",
        "49"
      ],
      "tasks_worse": [
        "3",
        "14",
        "18",
        "19",
        "23",
        "24",
        "30",
        "32",
        "42",
        "45"
      ],
      "tasks_tied": 33,
      "perfect_task_discordance": {
        "gains": 4,
        "losses": 5,
        "both_perfect": 24,
        "exact_mcnemar_two_sided_p": 1.0
      },
      "conversation_discordance": {
        "gains": 16,
        "losses": 20,
        "changed_outcomes": 36
      }
    },
    "mcp_chat_minus_native_mean": {
      "difference_pp": [
        -3.5000000000000004,
        -5.5,
        -5.5,
        -5.0
      ],
      "task_bootstrap_ci95_pp": [
        [
          -7.5,
          0.25
        ],
        [
          -11.66666666666667,
          0.6666666666666665
        ],
        [
          -14.249999999999998,
          3.0
        ],
        [
          -16.0,
          6.0
        ]
      ],
      "task_bootstrap_ci90_pp": [
        [
          -6.75,
          -0.25
        ],
        [
          -10.666666666666668,
          -0.3333333333333335
        ],
        [
          -12.75,
          1.5
        ],
        [
          -14.000000000000002,
          4.0
        ]
      ],
      "task_standard_error_pp": [
        1.9897697538834451,
        3.173104805125162,
        4.419922429829995,
        5.758755534498967
      ],
      "tasks_better": [
        "1",
        "9",
        "12",
        "29",
        "31",
        "37",
        "39",
        "41",
        "48",
        "49"
      ],
      "tasks_worse": [
        "3",
        "6",
        "14",
        "17",
        "18",
        "19",
        "20",
        "23",
        "24",
        "25",
        "30",
        "32",
        "33",
        "42",
        "43",
        "45"
      ],
      "tasks_tied": 24
    },
    "responses_minus_native_mean": {
      "difference_pp": [
        0.5,
        1.1666666666666665,
        2.5,
        5.0
      ],
      "task_bootstrap_ci95_pp": [
        [
          -3.75,
          4.75
        ],
        [
          -5.833333333333336,
          8.166666666666666
        ],
        [
          -6.5,
          11.5
        ],
        [
          -6.0,
          15.0
        ]
      ],
      "task_bootstrap_ci90_pp": [
        [
          -3.25,
          4.25
        ],
        [
          -4.666666666666668,
          7.000000000000001
        ],
        [
          -5.0,
          10.0
        ],
        [
          -4.0,
          14.000000000000002
        ]
      ],
      "task_standard_error_pp": [
        2.2292123620467477,
        3.5952380952380945,
        4.615302851185744,
        5.392738882336251
      ],
      "tasks_better": [
        "1",
        "15",
        "17",
        "20",
        "24",
        "25",
        "29",
        "33",
        "42",
        "43",
        "45",
        "48",
        "49"
      ],
      "tasks_worse": [
        "6",
        "9",
        "12",
        "19",
        "22",
        "31",
        "32",
        "35",
        "37",
        "41"
      ],
      "tasks_tied": 27
    }
  },
  "native_conditional_regrouping": {
    "native_eight_trial_success_distribution": {
      "0": 3,
      "1": 2,
      "4": 1,
      "5": 4,
      "6": 4,
      "7": 12,
      "8": 24
    },
    "expected_perfect_tasks": 31.15714285714286,
    "single_four_trial_block_perfect_tasks_ci95": [
      27,
      35
    ],
    "two_blocks_difference_perfect_tasks_ci95": [
      -7,
      7
    ],
    "prob_absolute_gap_ge_3_tasks_6pp": 0.5129939469458695,
    "prob_absolute_gap_ge_4_tasks_8pp": 0.3587762175158456,
    "prob_single_block_at_most_28_perfect": 0.0877086600107234,
    "assumptions": "Condition on native observed outcomes; exchangeable trials within each task; independent regrouping across tasks. Does not measure future run variability or prove MCP equivalence."
  },
  "worse_than_both_native_tasks": [
    "3",
    "14",
    "18",
    "19",
    "23",
    "24",
    "30",
    "45"
  ],
  "failures_in_worse_both_subset": 14,
  "budget": {
    "native_batch": 22.66237925,
    "mcp_batch": 21.1300026,
    "one_new_pair": 43.79238185
  },
  "illustrative_noninferiority_planning": [
    {
      "metric": "pass1",
      "margin_pp": 5.0,
      "assumed_true_loss_pp": 0.0,
      "units_per_arm": 792,
      "four_trial_batches_per_arm": 4,
      "total_budget_at_observed_prices": 175.1695274
    },
    {
      "metric": "pass1",
      "margin_pp": 5.0,
      "assumed_true_loss_pp": 3.5000000000000004,
      "units_per_arm": 8793,
      "four_trial_batches_per_arm": 44,
      "total_budget_at_observed_prices": 1926.8648014
    },
    {
      "metric": "pass1",
      "margin_pp": 10.0,
      "assumed_true_loss_pp": 0.0,
      "units_per_arm": 198,
      "four_trial_batches_per_arm": 1,
      "total_budget_at_observed_prices": 43.79238185
    },
    {
      "metric": "pass1",
      "margin_pp": 10.0,
      "assumed_true_loss_pp": 3.5000000000000004,
      "units_per_arm": 469,
      "four_trial_batches_per_arm": 3,
      "total_budget_at_observed_prices": 131.37714555
    },
    {
      "metric": "pass4",
      "margin_pp": 5.0,
      "assumed_true_loss_pp": 0.0,
      "units_per_arm": 1188,
      "four_trial_batches_per_arm": 24,
      "total_budget_at_observed_prices": 1051.0171644
    },
    {
      "metric": "pass4",
      "margin_pp": 5.0,
      "assumed_true_loss_pp": 5.0,
      "units_per_arm": null,
      "four_trial_batches_per_arm": null,
      "total_budget_at_observed_prices": null
    },
    {
      "metric": "pass4",
      "margin_pp": 10.0,
      "assumed_true_loss_pp": 0.0,
      "units_per_arm": 297,
      "four_trial_batches_per_arm": 6,
      "total_budget_at_observed_prices": 262.7542911
    },
    {
      "metric": "pass4",
      "margin_pp": 10.0,
      "assumed_true_loss_pp": 5.0,
      "units_per_arm": 1188,
      "four_trial_batches_per_arm": 24,
      "total_budget_at_observed_prices": 1051.0171644
    }
  ],
  "planning_assumptions": "Independent Bernoulli evaluation units, stable performance, no task pairing benefit or shared run shocks, normal approximation, one prespecified endpoint. Fixed tasks, positive pairing, heterogeneity and shared provider effects can change required size substantially. Counts are hypothetical total future balanced batches; no credit assigned to exploratory existing data.",
  "integrity": "Inputs verified binary rewards, 200 unique task/trial pairs, 50 tasks, 4 trials, identical matched saved policy/simulator/model settings and seeds. This script does not repeat strict environment rescoring."
}