{
  "benchmark": "codex_ws_streaming",
  "benchmark_version": "1",
  "date": "2026-05-24",
  "endpoint": "wss://chatgpt.com/backend-api/codex/responses",
  "model": "gpt-5.4-mini",
  "warmup_count": 1,
  "sample_count": 6,
  "inter_request_delay_ms": 2000,
  "lead_alternation": "SR/RS/SR",
  "primary_claim": "pass",
  "primary_claim_passed": 6,
  "primary_claim_total": 6,
  "primary_claim_threshold": 5,
  "primary_claim_definition": "WebSocket connect, send response.create, receive >=1 response.output_text.delta AND response.completed text frame within 30s; pass if >=ceil(0.8*N) of N samples (counted per-pair, both clients must pass)",
  "failures": [],
  "environment": {
    "os": "macos",
    "arch": "aarch64",
    "specter_version": "3.2.0",
    "tokio_tungstenite_version": "0.24"
  },
  "rows": [
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 306.001167,
      "ttft_ms": 1320.5225,
      "total_wall_time_ms": 1320.661125,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 48000.0,
      "epoch_ms": 1779656711694
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 488.082,
      "ttft_ms": 2784.504958,
      "total_wall_time_ms": 2839.624958,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 870.8272859216272,
      "epoch_ms": 1779656717062
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 275.67362499999996,
      "ttft_ms": 502.76500000000004,
      "total_wall_time_ms": 625.7247500000001,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 390.3716460061116,
      "epoch_ms": 1779656720077
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 355.682792,
      "ttft_ms": 1391.302875,
      "total_wall_time_ms": 1433.9458330000002,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1547.7350328276898,
      "epoch_ms": 1779656723892
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 320.951667,
      "ttft_ms": 533.107875,
      "total_wall_time_ms": 679.186084,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 328.59110423512925,
      "epoch_ms": 1779656726909
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 364.754084,
      "ttft_ms": 554.8841669999999,
      "total_wall_time_ms": 654.692042,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 480.9239751873282,
      "epoch_ms": 1779656730045
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 366.322708,
      "ttft_ms": 769.117666,
      "total_wall_time_ms": 802.097666,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 1455.4275318374766,
      "epoch_ms": 1779656733274
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 298.77379199999996,
      "ttft_ms": 1047.0543750000002,
      "total_wall_time_ms": 1100.4073329999999,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1237.0448138976728,
      "epoch_ms": 1779656736686
    }
  ],
  "summary": {
    "specter": {
      "median_ttft_ms": 554.8841669999999,
      "p95_ttft_ms": 747.6943160999999,
      "median_total_wall_time_ms": 654.692042,
      "p95_total_wall_time_ms": 787.3571036,
      "median_handshake_ms": 364.754084,
      "median_chars_per_sec": 480.9239751873282,
      "passing_samples": 3
    },
    "tungstenite": {
      "median_ttft_ms": 1047.0543750000002,
      "p95_ttft_ms": 1356.878025,
      "median_total_wall_time_ms": 1100.4073329999999,
      "p95_total_wall_time_ms": 1400.591983,
      "median_handshake_ms": 320.951667,
      "median_chars_per_sec": 1237.0448138976728,
      "passing_samples": 3
    },
    "comparison": {
      "ttft_difference_ms": -381.5660973333334,
      "ttft_ci_95": [
        -1534.1015959041888,
        770.9694012375221
      ],
      "ttft_ci_covers_zero": true,
      "wall_time_difference_ms": -377.00826400000005,
      "wall_time_ci_95": [
        -1365.1445987196953,
        611.1280707196952
      ],
      "wall_time_ci_covers_zero": true,
      "handshake_difference_ms": 10.447388666666654,
      "handshake_ci_95": [
        -186.392567568228,
        207.2873449015613
      ],
      "handshake_ci_covers_zero": true,
      "ttft_wilcoxon_p_value": 0.21133897748577657,
      "wall_time_wilcoxon_p_value": 0.09072466857072525,
      "interpretation": "All differences within network noise at n=3. Both clients streamed successfully from Codex WS."
    }
  }
}