{
  "benchmark": "codex_real_streaming",
  "benchmark_version": "1",
  "date": "2026-05-24",
  "endpoint": "https://chatgpt.com/backend-api/codex/responses",
  "model": "gpt-5.4-mini",
  "accept_encoding": "identity",
  "warmup_count": 1,
  "sample_count": 10,
  "inter_request_delay_ms": 2000,
  "lead_alternation": "SR/RS/SR/RS/SR",
  "primary_claim": "pass",
  "primary_claim_passed": 10,
  "primary_claim_total": 10,
  "primary_claim_threshold": 8,
  "primary_claim_definition": "status_code==200 AND delta_count>=1 AND response.completed received within 30s; pass if >=ceil(0.8*N) of N samples meet condition (counted per-pair, both clients must pass for the pair to count)",
  "failures": [],
  "environment": {
    "os": "macos",
    "arch": "aarch64",
    "specter_version": "3.2.0"
  },
  "protocol_mismatch": false,
  "rows": [
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 559.925708,
      "total_wall_time_ms": 660.4339580000001,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 477.57273656640086,
      "pool_reuse_delta_specter": 0,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656815493
    },
    {
      "client": "reqwest",
      "warmup": true,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 727.4043340000001,
      "total_wall_time_ms": 824.811292,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 492.7779389230084,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656818400
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 558.7953329999999,
      "total_wall_time_ms": 670.698083,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 428.94388207617743,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656821172
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 578.9905,
      "total_wall_time_ms": 740.687042,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 296.8523594029611,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656823985
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 930.1407499999999,
      "total_wall_time_ms": 1040.36725,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 435.466970283915,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656827026
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 625.135875,
      "total_wall_time_ms": 717.7531670000001,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 518.2617518119617,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656829824
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 551.5627920000001,
      "total_wall_time_ms": 643.1352919999999,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 524.1748341478071,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656832541
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 924.393291,
      "total_wall_time_ms": 968.917083,
      "total_chars": 66,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 1482.3535246054491,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656835570
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 6,
      "pair_index": 3,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 1041.348667,
      "total_wall_time_ms": 1087.5385419999998,
      "total_chars": 66,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 1428.8845769771033,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656838718
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 7,
      "pair_index": 3,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 597.064084,
      "total_wall_time_ms": 708.60225,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 430.34596785462634,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656841494
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 8,
      "pair_index": 4,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 531.023833,
      "total_wall_time_ms": 636.88075,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 453.4422630124394,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656844230
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 9,
      "pair_index": 4,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 560.4407500000001,
      "total_wall_time_ms": 717.6911249999999,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 305.2456949625719,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656846980
    }
  ],
  "summary": {
    "specter": {
      "median_ttft_ms": 558.7953329999999,
      "p95_ttft_ms": 619.5215168000001,
      "median_total_wall_time_ms": 670.698083,
      "p95_total_wall_time_ms": 715.9229836000001,
      "median_chars_per_sec": 453.4422630124394,
      "passing_samples": 5
    },
    "reqwest": {
      "median_ttft_ms": 924.393291,
      "p95_ttft_ms": 1019.1070835999999,
      "median_total_wall_time_ms": 968.917083,
      "p95_total_wall_time_ms": 1078.1042835999997,
      "median_chars_per_sec": 435.466970283915,
      "passing_samples": 5
    },
    "protocol_used_specter": "HTTP/2",
    "protocol_used_reqwest": "HTTP/2",
    "protocol_mismatch": false,
    "comparison": {
      "ttft_difference_ms": -234.34640819999996,
      "ttft_ci_95": [
        -479.59571780974284,
        10.902901409742924
      ],
      "ttft_ci_covers_zero": true,
      "wall_time_difference_ms": -235.6263,
      "wall_time_ci_95": [
        -419.3844373685183,
        -51.868162631481624
      ],
      "wall_time_ci_covers_zero": false,
      "ttft_wilcoxon_p_value": 0.02952904975182835,
      "wall_time_wilcoxon_p_value": 0.02952904975182835,
      "interpretation": "Mixed: TTFT CI=[-479.6, 10.9], wall CI=[-419.4, -51.9]"
    }
  }
}