{
  "benchmark": "codex_ws_streaming",
  "benchmark_version": "1",
  "date": "2026-05-24",
  "endpoint": "wss://chatgpt.com/backend-api/codex/responses",
  "model": "gpt-5.4-mini",
  "warmup_count": 4,
  "sample_count": 30,
  "inter_request_delay_ms": 2000,
  "lead_alternation": "SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR",
  "primary_claim": "pass",
  "primary_claim_passed": 30,
  "primary_claim_total": 30,
  "primary_claim_threshold": 24,
  "primary_claim_definition": "WebSocket connect, send response.create, receive >=1 response.output_text.delta AND response.completed text frame within 30s; pass if >=ceil(0.8*N) of N samples (counted per-pair, both clients must pass)",
  "failures": [],
  "environment": {
    "os": "macos",
    "arch": "aarch64",
    "specter_version": "3.2.0",
    "tokio_tungstenite_version": "0.24"
  },
  "rows": [
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 347.388333,
      "ttft_ms": 511.020625,
      "total_wall_time_ms": 648.6850830000001,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 348.6738748501082,
      "epoch_ms": 1779657212372
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 381.394041,
      "ttft_ms": 1040.251583,
      "total_wall_time_ms": 1080.761833,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1184.8853068050666,
      "epoch_ms": 1779657215855
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 281.301458,
      "ttft_ms": 1987.057041,
      "total_wall_time_ms": 2620.265666,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 104.231050232457,
      "epoch_ms": 1779657220824
    },
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 297.438084,
      "ttft_ms": 802.348667,
      "total_wall_time_ms": 837.049625,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1901.9647814910463,
      "epoch_ms": 1779657223987
    },
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 296.685166,
      "ttft_ms": 781.181417,
      "total_wall_time_ms": 829.6481249999999,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1361.7594989121212,
      "epoch_ms": 1779657227125
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 270.61337499999996,
      "ttft_ms": 556.994042,
      "total_wall_time_ms": 663.439375,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 450.9356929720911,
      "epoch_ms": 1779657230097
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 6,
      "pair_index": 3,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 293.520291,
      "ttft_ms": 3052.730458,
      "total_wall_time_ms": 3097.175417,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1079.9874964447617,
      "epoch_ms": 1779657235516
    },
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 7,
      "pair_index": 3,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 374.96562500000005,
      "ttft_ms": 1057.221334,
      "total_wall_time_ms": 1107.660959,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1308.495057209486,
      "epoch_ms": 1779657239001
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 308.534042,
      "ttft_ms": 564.869208,
      "total_wall_time_ms": 678.119125,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 423.841370232527,
      "epoch_ms": 1779657242031
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 312.112958,
      "ttft_ms": 608.430083,
      "total_wall_time_ms": 656.707,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1367.1129827946545,
      "epoch_ms": 1779657245035
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 328.427209,
      "ttft_ms": 526.2926669999999,
      "total_wall_time_ms": 646.568125,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 399.08390953705594,
      "epoch_ms": 1779657248039
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 343.074333,
      "ttft_ms": 399.600416,
      "total_wall_time_ms": 566.0376660000001,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 288.39697844082366,
      "epoch_ms": 1779657250951
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 308.633125,
      "ttft_ms": 848.991375,
      "total_wall_time_ms": 1053.329875,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 322.99346427618883,
      "epoch_ms": 1779657254315
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 365.376625,
      "ttft_ms": 1135.932167,
      "total_wall_time_ms": 1379.374792,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 197.1717155120225,
      "epoch_ms": 1779657258119
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 6,
      "pair_index": 3,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 294.066,
      "ttft_ms": 873.487542,
      "total_wall_time_ms": 1002.748459,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 371.3419424372487,
      "epoch_ms": 1779657261483
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 7,
      "pair_index": 3,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 287.327917,
      "ttft_ms": 522.1443340000001,
      "total_wall_time_ms": 636.0331249999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 421.46377688740296,
      "epoch_ms": 1779657264448
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 8,
      "pair_index": 4,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 325.89833400000003,
      "ttft_ms": 487.487417,
      "total_wall_time_ms": 635.052792,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 325.2795582974665,
      "epoch_ms": 1779657267412
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 9,
      "pair_index": 4,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 380.74449999999996,
      "ttft_ms": 582.1409170000001,
      "total_wall_time_ms": 662.529167,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 597.102188441719,
      "epoch_ms": 1779657270554
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 10,
      "pair_index": 5,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 651.109708,
      "ttft_ms": 533.0572500000001,
      "total_wall_time_ms": 634.314541,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 651.8049154603599,
      "epoch_ms": 1779657273883
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 11,
      "pair_index": 5,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 345.46629099999996,
      "ttft_ms": 622.3585830000001,
      "total_wall_time_ms": 622.432417,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 66000.0,
      "epoch_ms": 1779657276894
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 12,
      "pair_index": 6,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 362.70383400000003,
      "ttft_ms": 538.38325,
      "total_wall_time_ms": 743.19475,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 234.36184003339653,
      "epoch_ms": 1779657280002
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 13,
      "pair_index": 6,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 353.145375,
      "ttft_ms": 789.4452090000001,
      "total_wall_time_ms": 844.241,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 1204.4720734116263,
      "epoch_ms": 1779657283232
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 14,
      "pair_index": 7,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 289.91654100000005,
      "ttft_ms": 626.07325,
      "total_wall_time_ms": 769.825042,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 333.90888094111546,
      "epoch_ms": 1779657286295
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 15,
      "pair_index": 7,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 424.33829099999997,
      "ttft_ms": 492.146417,
      "total_wall_time_ms": 597.9249169999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 453.7784143280536,
      "epoch_ms": 1779657289439
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 16,
      "pair_index": 8,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 391.61400000000003,
      "ttft_ms": 629.013667,
      "total_wall_time_ms": 752.781959,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 533.2545107756679,
      "epoch_ms": 1779657292642
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 17,
      "pair_index": 8,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 336.187167,
      "ttft_ms": 909.93625,
      "total_wall_time_ms": 956.388958,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1420.799837977152,
      "epoch_ms": 1779657295945
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 18,
      "pair_index": 9,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 271.644208,
      "ttft_ms": 541.211833,
      "total_wall_time_ms": 654.938833,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 422.06336226225926,
      "epoch_ms": 1779657298898
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 19,
      "pair_index": 9,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 332.317125,
      "ttft_ms": 649.957583,
      "total_wall_time_ms": 790.7364580000001,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 468.8203396994042,
      "epoch_ms": 1779657302024
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 20,
      "pair_index": 10,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 357.867292,
      "ttft_ms": 597.485625,
      "total_wall_time_ms": 783.3462079999999,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 355.1048798765472,
      "epoch_ms": 1779657305167
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 21,
      "pair_index": 10,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 433.166458,
      "ttft_ms": 493.442916,
      "total_wall_time_ms": 590.278,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 681.571154520814,
      "epoch_ms": 1779657308311
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 22,
      "pair_index": 11,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 436.42674999999997,
      "ttft_ms": 574.932833,
      "total_wall_time_ms": 707.983625,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 496.0511621757201,
      "epoch_ms": 1779657311503
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 23,
      "pair_index": 11,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 329.183625,
      "ttft_ms": 955.92325,
      "total_wall_time_ms": 1005.1827499999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 974.4313279671964,
      "epoch_ms": 1779657314880
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 24,
      "pair_index": 12,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 344.684416,
      "ttft_ms": 857.204125,
      "total_wall_time_ms": 902.4952499999999,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1457.239139014543,
      "epoch_ms": 1779657318164
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 25,
      "pair_index": 12,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 281.07275,
      "ttft_ms": 644.1885000000001,
      "total_wall_time_ms": 715.045875,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 931.4485612824362,
      "epoch_ms": 1779657321188
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 26,
      "pair_index": 13,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 326.566292,
      "ttft_ms": 680.792458,
      "total_wall_time_ms": 730.597041,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1325.1792510741436,
      "epoch_ms": 1779657324283
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 27,
      "pair_index": 13,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 374.977083,
      "ttft_ms": 1115.103625,
      "total_wall_time_ms": 1215.503625,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 478.08764940239,
      "epoch_ms": 1779657327932
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 28,
      "pair_index": 14,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 265.609875,
      "ttft_ms": 795.320958,
      "total_wall_time_ms": 844.0667910000001,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1353.96188634216,
      "epoch_ms": 1779657331052
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 29,
      "pair_index": 14,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 275.741833,
      "ttft_ms": 5392.9285,
      "total_wall_time_ms": 10015.015,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 10.38492031683094,
      "epoch_ms": 1779657345074
    }
  ],
  "summary": {
    "specter": {
      "median_ttft_ms": 622.3585830000001,
      "p95_ttft_ms": 1003.6773624999998,
      "median_total_wall_time_ms": 752.781959,
      "p95_total_wall_time_ms": 1101.9819999999997,
      "median_handshake_ms": 343.074333,
      "median_chars_per_sec": 453.7784143280536,
      "passing_samples": 15
    },
    "tungstenite": {
      "median_ttft_ms": 626.07325,
      "p95_ttft_ms": 2413.0310668999955,
      "median_total_wall_time_ms": 715.045875,
      "p95_total_wall_time_ms": 3970.0668543999905,
      "median_handshake_ms": 328.427209,
      "median_chars_per_sec": 597.102188441719,
      "passing_samples": 15
    },
    "comparison": {
      "ttft_difference_ms": -322.4201694666666,
      "ttft_ci_95": [
        -990.657391059648,
        345.8170521263148
      ],
      "ttft_ci_covers_zero": true,
      "wall_time_difference_ms": -609.3545166666667,
      "wall_time_ci_95": [
        -1927.312404370739,
        708.6033710374055
      ],
      "wall_time_ci_covers_zero": true,
      "handshake_difference_ms": -15.56498606666665,
      "handshake_ci_95": [
        -71.90217823229375,
        40.772206098960446
      ],
      "handshake_ci_covers_zero": true,
      "ttft_wilcoxon_p_value": 0.22161547183156394,
      "wall_time_wilcoxon_p_value": 0.2947489413971336,
      "interpretation": "All differences within network noise at n=15. Both clients streamed successfully from Codex WS."
    }
  }
}