{
  "benchmark": "codex_real_streaming",
  "benchmark_version": "1",
  "date": "2026-05-24",
  "endpoint": "https://chatgpt.com/backend-api/codex/responses",
  "model": "gpt-5.4-mini",
  "accept_encoding": "identity",
  "warmup_count": 1,
  "sample_count": 6,
  "inter_request_delay_ms": 2000,
  "lead_alternation": "SR/RS/SR",
  "primary_claim": "fail",
  "primary_claim_passed": 6,
  "primary_claim_total": 6,
  "primary_claim_threshold": 8,
  "primary_claim_definition": "status_code==200 AND delta_count>=1 AND response.completed received within 30s; pass if >=8 of N samples meet condition (counted per-pair, both clients must pass for the pair to count)",
  "failures": [],
  "environment": {
    "os": "macos",
    "arch": "aarch64",
    "specter_version": "3.2.0"
  },
  "protocol_mismatch": false,
  "rows": [
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 1468.54925,
      "total_wall_time_ms": 1537.537875,
      "total_chars": 66,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 956.6794525909169,
      "pool_reuse_delta_specter": 0,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656333840
    },
    {
      "client": "reqwest",
      "warmup": true,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 624.352083,
      "total_wall_time_ms": 704.331958,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 600.1509754797692,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656336611
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 959.1777500000001,
      "total_wall_time_ms": 1007.719708,
      "total_chars": 66,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 1359.6484921354042,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656339652
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 950.135708,
      "total_wall_time_ms": 1044.4032080000002,
      "total_chars": 66,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 700.1352534012241,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656342749
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 559.226792,
      "total_wall_time_ms": 658.8260839999999,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 481.9311366189235,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656345495
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 612.644542,
      "total_wall_time_ms": 717.3969169999999,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 458.22350089914465,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656348370
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 1143.314042,
      "total_wall_time_ms": 1143.418375,
      "total_chars": 48,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 48000.0,
      "pool_reuse_delta_specter": 1,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656351582
    },
    {
      "client": "reqwest",
      "warmup": false,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "status_code": 200,
      "ttft_ms": 1165.085833,
      "total_wall_time_ms": 1206.160291,
      "total_chars": 66,
      "delta_count": 19,
      "completed": true,
      "chars_per_sec": 1606.8380013681565,
      "pool_reuse_delta_specter": null,
      "protocol_used": "HTTP/2",
      "epoch_ms": 1779656354843
    }
  ],
  "summary": {
    "specter": {
      "median_ttft_ms": 959.1777500000001,
      "p95_ttft_ms": 1124.9004128,
      "median_total_wall_time_ms": 1007.719708,
      "p95_total_wall_time_ms": 1129.8485083,
      "median_chars_per_sec": 1359.6484921354042,
      "passing_samples": 3
    },
    "reqwest": {
      "median_ttft_ms": 950.135708,
      "p95_ttft_ms": 1143.5908205,
      "median_total_wall_time_ms": 1044.4032080000002,
      "p95_total_wall_time_ms": 1189.9845827,
      "median_chars_per_sec": 700.1352534012241,
      "passing_samples": 3
    },
    "protocol_used_specter": "HTTP/2",
    "protocol_used_reqwest": "HTTP/2",
    "protocol_mismatch": false,
    "comparison": {
      "ttft_difference_ms": 13.562666999999957,
      "ttft_ci_95": [
        -80.34051300489058,
        107.4658470048905
      ],
      "ttft_ci_covers_zero": true,
      "wall_time_difference_ms": -13.618194333333387,
      "wall_time_ci_95": [
        -172.2700243577906,
        145.03363569112383
      ],
      "wall_time_ci_covers_zero": true,
      "ttft_wilcoxon_p_value": 0.7886610225142234,
      "wall_time_wilcoxon_p_value": 0.3946340746214716,
      "interpretation": "Differences within network noise at n=3 (TTFT 95% CI covers zero, wall 95% CI covers zero). Both clients streamed successfully from Codex over HTTP/2."
    }
  }
}