{
  "benchmark": "codex_ws_streaming",
  "benchmark_version": "1",
  "date": "2026-05-24",
  "endpoint": "wss://chatgpt.com/backend-api/codex/responses",
  "model": "gpt-5.4-mini",
  "warmup_count": 1,
  "sample_count": 10,
  "inter_request_delay_ms": 2000,
  "lead_alternation": "SR/RS/SR/RS/SR",
  "primary_claim": "pass",
  "primary_claim_passed": 10,
  "primary_claim_total": 10,
  "primary_claim_threshold": 8,
  "primary_claim_definition": "WebSocket connect, send response.create, receive >=1 response.output_text.delta AND response.completed text frame within 30s; pass if >=ceil(0.8*N) of N samples (counted per-pair, both clients must pass)",
  "failures": [],
  "environment": {
    "os": "macos",
    "arch": "aarch64",
    "specter_version": "3.2.0",
    "tokio_tungstenite_version": "0.24"
  },
  "rows": [
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 276.826417,
      "ttft_ms": 872.6584160000001,
      "total_wall_time_ms": 948.361708,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 871.8247021543009,
      "epoch_ms": 1779656763426
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 326.65925,
      "ttft_ms": 870.9649579999999,
      "total_wall_time_ms": 987.0450830000001,
      "total_chars": 57,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 491.04013283927725,
      "epoch_ms": 1779656766743
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 382.296292,
      "ttft_ms": 956.997,
      "total_wall_time_ms": 1001.8789589999999,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1470.5240473126428,
      "epoch_ms": 1779656770154
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 391.57470900000004,
      "ttft_ms": 518.862666,
      "total_wall_time_ms": 619.671458,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 476.1489454213476,
      "epoch_ms": 1779656773202
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 330.541042,
      "ttft_ms": 988.818625,
      "total_wall_time_ms": 1053.517083,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 741.9033078037197,
      "epoch_ms": 1779656776698
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 406.633125,
      "ttft_ms": 546.9919170000001,
      "total_wall_time_ms": 637.6072499999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 529.7116769410325,
      "epoch_ms": 1779656779842
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 378.943166,
      "ttft_ms": 558.8975,
      "total_wall_time_ms": 671.126875,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 427.69551198160013,
      "epoch_ms": 1779656782989
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 382.96175,
      "ttft_ms": 545.487834,
      "total_wall_time_ms": 655.3312500000001,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 436.9856815086667,
      "epoch_ms": 1779656786137
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 6,
      "pair_index": 3,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 471.421875,
      "ttft_ms": 446.629417,
      "total_wall_time_ms": 542.70225,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 499.6209490356131,
      "epoch_ms": 1779656789283
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 7,
      "pair_index": 3,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 392.192791,
      "ttft_ms": 796.2658329999999,
      "total_wall_time_ms": 842.375,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1041.00774581332,
      "epoch_ms": 1779656792539
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 8,
      "pair_index": 4,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 309.355292,
      "ttft_ms": 533.622166,
      "total_wall_time_ms": 724.243208,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 251.8085070587328,
      "epoch_ms": 1779656795575
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 9,
      "pair_index": 4,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 373.38100000000003,
      "ttft_ms": 767.5918750000001,
      "total_wall_time_ms": 767.6468749999999,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 66000.0,
      "epoch_ms": 1779656798725
    }
  ],
  "summary": {
    "specter": {
      "median_ttft_ms": 558.8975,
      "p95_ttft_ms": 924.8507665999999,
      "median_total_wall_time_ms": 724.243208,
      "p95_total_wall_time_ms": 969.9781671999999,
      "median_handshake_ms": 382.296292,
      "median_chars_per_sec": 529.7116769410325,
      "passing_samples": 5
    },
    "tungstenite": {
      "median_ttft_ms": 545.487834,
      "p95_ttft_ms": 944.573275,
      "median_total_wall_time_ms": 655.3312500000001,
      "p95_total_wall_time_ms": 996.3430414,
      "median_handshake_ms": 382.96175,
      "median_chars_per_sec": 499.6209490356131,
      "passing_samples": 5
    },
    "comparison": {
      "ttft_difference_ms": 25.076799799999982,
      "ttft_ci_95": [
        -439.9324394089911,
        490.0860390089911
      ],
      "ttft_ci_covers_zero": true,
      "wall_time_difference_ms": 47.672475199999965,
      "wall_time_ci_95": [
        -344.7352179884089,
        440.0801683884088
      ],
      "wall_time_ci_covers_zero": true,
      "handshake_difference_ms": -16.091942000000007,
      "handshake_ci_95": [
        -92.04999105474846,
        59.866107054748454
      ],
      "handshake_ci_covers_zero": true,
      "ttft_wilcoxon_p_value": 0.6062966923611464,
      "wall_time_wilcoxon_p_value": 0.6062966923611464,
      "interpretation": "All differences within network noise at n=5. Both clients streamed successfully from Codex WS."
    }
  }
}