{
  "benchmark": "codex_ws_streaming",
  "benchmark_version": "1",
  "date": "2026-05-24",
  "endpoint": "wss://chatgpt.com/backend-api/codex/responses",
  "model": "gpt-5.4-mini",
  "warmup_count": 4,
  "sample_count": 30,
  "inter_request_delay_ms": 2000,
  "lead_alternation": "SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR",
  "primary_claim": "pass",
  "primary_claim_passed": 30,
  "primary_claim_total": 30,
  "primary_claim_threshold": 24,
  "primary_claim_definition": "WebSocket connect, send response.create, receive >=1 response.output_text.delta AND response.completed text frame within 30s; pass if >=ceil(0.8*N) of N samples (counted per-pair, both clients must pass)",
  "failures": [],
  "environment": {
    "os": "macos",
    "arch": "aarch64",
    "specter_version": "4.0.0",
    "tokio_tungstenite_version": "0.24"
  },
  "rows": [
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 359.980167,
      "ttft_ms": 558.5572920000001,
      "total_wall_time_ms": 671.0193330000001,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 426.8106782803275,
      "epoch_ms": 1779657900923
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 261.34545799999995,
      "ttft_ms": 703.967875,
      "total_wall_time_ms": 704.0456670000001,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 66000.0,
      "epoch_ms": 1779657903894
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 479.29029099999997,
      "ttft_ms": 940.354542,
      "total_wall_time_ms": 983.193167,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1120.4841425232491,
      "epoch_ms": 1779657907384
    },
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 330.09325,
      "ttft_ms": 476.756625,
      "total_wall_time_ms": 576.4121250000001,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 662.2815599741103,
      "epoch_ms": 1779657910335
    },
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 263.873958,
      "ttft_ms": 856.349417,
      "total_wall_time_ms": 898.26675,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1574.5276542283837,
      "epoch_ms": 1779657913532
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 432.60425000000004,
      "ttft_ms": 842.1440839999999,
      "total_wall_time_ms": 888.8922500000001,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1411.8200915090385,
      "epoch_ms": 1779657916890
    },
    {
      "client": "tungstenite",
      "warmup": true,
      "sample_index": 6,
      "pair_index": 3,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 342.800208,
      "ttft_ms": 551.6554580000001,
      "total_wall_time_ms": 650.163042,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 487.27212718972,
      "epoch_ms": 1779657919926
    },
    {
      "client": "specter",
      "warmup": true,
      "sample_index": 7,
      "pair_index": 3,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 482.838458,
      "ttft_ms": 1037.436291,
      "total_wall_time_ms": 1088.115458,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1302.31027672574,
      "epoch_ms": 1779657923501
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 0,
      "pair_index": 0,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 302.754166,
      "ttft_ms": 834.534167,
      "total_wall_time_ms": 879.957,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1056.7372581098152,
      "epoch_ms": 1779657926689
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 1,
      "pair_index": 0,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 363.74925,
      "ttft_ms": 923.9611669999999,
      "total_wall_time_ms": 1960.0183339999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 46.32948984754237,
      "epoch_ms": 1779657931780
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 2,
      "pair_index": 1,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 323.477958,
      "ttft_ms": 607.508333,
      "total_wall_time_ms": 859.2747499999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 190.6529098358659,
      "epoch_ms": 1779657934970
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 3,
      "pair_index": 1,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 277.633334,
      "ttft_ms": 996.3853750000001,
      "total_wall_time_ms": 1023.348375,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 2447.798835441163,
      "epoch_ms": 1779657938295
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 4,
      "pair_index": 2,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 448.446125,
      "ttft_ms": 548.812542,
      "total_wall_time_ms": 640.07875,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 525.933979858131,
      "epoch_ms": 1779657941424
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 5,
      "pair_index": 2,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 312.74583299999995,
      "ttft_ms": 1045.096333,
      "total_wall_time_ms": 1045.147458,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 48000.0,
      "epoch_ms": 1779657944804
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 6,
      "pair_index": 3,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 491.58275000000003,
      "ttft_ms": 962.88,
      "total_wall_time_ms": 972.208666,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 7074.966560063359,
      "epoch_ms": 1779657948298
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 7,
      "pair_index": 3,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 295.52858399999997,
      "ttft_ms": 1200.877792,
      "total_wall_time_ms": 1209.3378750000002,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 5673.703201256896,
      "epoch_ms": 1779657951867
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 8,
      "pair_index": 4,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 352.741792,
      "ttft_ms": 423.259333,
      "total_wall_time_ms": 525.760416,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 643.8956357173323,
      "epoch_ms": 1779657954783
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 9,
      "pair_index": 4,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 339.68795800000004,
      "ttft_ms": 489.765833,
      "total_wall_time_ms": 601.167958,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 430.87149369906535,
      "epoch_ms": 1779657957764
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 10,
      "pair_index": 5,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 282.28033400000004,
      "ttft_ms": 655.4414589999999,
      "total_wall_time_ms": 705.6816249999999,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1313.68992690032,
      "epoch_ms": 1779657960767
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 11,
      "pair_index": 5,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 364.12575000000004,
      "ttft_ms": 708.686709,
      "total_wall_time_ms": 731.52725,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 2101.5264043001407,
      "epoch_ms": 1779657963903
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 12,
      "pair_index": 6,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 278.847042,
      "ttft_ms": 2410.487291,
      "total_wall_time_ms": 2475.858416,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1009.620073082724,
      "epoch_ms": 1779657968694
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 13,
      "pair_index": 6,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 301.055792,
      "ttft_ms": 839.4427089999999,
      "total_wall_time_ms": 881.2378749999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1148.458173368662,
      "epoch_ms": 1779657971908
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 14,
      "pair_index": 7,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 277.153166,
      "ttft_ms": 501.983125,
      "total_wall_time_ms": 589.198333,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 550.3627303164828,
      "epoch_ms": 1779657974820
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 15,
      "pair_index": 7,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 357.637792,
      "ttft_ms": 632.876459,
      "total_wall_time_ms": 632.920167,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 48000.0,
      "epoch_ms": 1779657977850
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 16,
      "pair_index": 8,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 343.525875,
      "ttft_ms": 462.809458,
      "total_wall_time_ms": 564.536291,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 471.8519055832594,
      "epoch_ms": 1779657980790
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 17,
      "pair_index": 8,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 441.631375,
      "ttft_ms": 2964.638834,
      "total_wall_time_ms": 2966.832334,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 21882.835650783294,
      "epoch_ms": 1779657986213
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 18,
      "pair_index": 9,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 352.192666,
      "ttft_ms": 778.752208,
      "total_wall_time_ms": 780.489417,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 37991.97448320826,
      "epoch_ms": 1779657989365
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 19,
      "pair_index": 9,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 341.218916,
      "ttft_ms": 586.219666,
      "total_wall_time_ms": 782.3876250000001,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 336.4463816438033,
      "epoch_ms": 1779657992491
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 20,
      "pair_index": 10,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 891.098417,
      "ttft_ms": 1013.3256250000001,
      "total_wall_time_ms": 1059.614208,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1425.8375548026634,
      "epoch_ms": 1779657996467
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 21,
      "pair_index": 10,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 319.368291,
      "ttft_ms": 550.087625,
      "total_wall_time_ms": 670.284416,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 399.3451039803552,
      "epoch_ms": 1779657999493
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 22,
      "pair_index": 11,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 435.27824999999996,
      "ttft_ms": 542.326167,
      "total_wall_time_ms": 574.082333,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 1511.5174797864504,
      "epoch_ms": 1779658002543
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 23,
      "pair_index": 11,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 412.78441599999996,
      "ttft_ms": 781.89875,
      "total_wall_time_ms": 832.526625,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 948.0943057554753,
      "epoch_ms": 1779658005806
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 24,
      "pair_index": 12,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 300.691875,
      "ttft_ms": 637.999375,
      "total_wall_time_ms": 671.6348340000001,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 1962.2149351373453,
      "epoch_ms": 1779658008813
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 25,
      "pair_index": 12,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 301.407,
      "ttft_ms": 469.925292,
      "total_wall_time_ms": 612.0917919999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 337.6322832734859,
      "epoch_ms": 1779658011769
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 26,
      "pair_index": 13,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 282.51741699999997,
      "ttft_ms": 981.366542,
      "total_wall_time_ms": 999.70225,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 30,
      "completed": true,
      "chars_per_sec": 3599.533762208678,
      "epoch_ms": 1779658015078
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 27,
      "pair_index": 13,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 328.480917,
      "ttft_ms": 429.56525,
      "total_wall_time_ms": 533.536875,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 461.6644204608709,
      "epoch_ms": 1779658017993
    },
    {
      "client": "specter",
      "warmup": false,
      "sample_index": 28,
      "pair_index": 14,
      "lead_in_pair": true,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 295.916292,
      "ttft_ms": 540.18575,
      "total_wall_time_ms": 623.498334,
      "total_chars": 66,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 792.1972507778656,
      "epoch_ms": 1779658020947
    },
    {
      "client": "tungstenite",
      "warmup": false,
      "sample_index": 29,
      "pair_index": 14,
      "lead_in_pair": false,
      "status": "ok",
      "handshake_status": 101,
      "handshake_ms": 304.215667,
      "ttft_ms": 694.080375,
      "total_wall_time_ms": 762.3283749999999,
      "total_chars": 48,
      "delta_count": 19,
      "frame_count": 28,
      "completed": true,
      "chars_per_sec": 703.3173133278638,
      "epoch_ms": 1779658024057
    }
  ],
  "summary": {
    "specter": {
      "median_ttft_ms": 637.999375,
      "p95_ttft_ms": 1563.7606416999988,
      "median_total_wall_time_ms": 731.52725,
      "p95_total_wall_time_ms": 1589.2940372999988,
      "median_handshake_ms": 341.218916,
      "median_chars_per_sec": 1009.620073082724,
      "passing_samples": 15
    },
    "tungstenite": {
      "median_ttft_ms": 694.080375,
      "p95_ttft_ms": 1620.959083299998,
      "median_total_wall_time_ms": 780.489417,
      "p95_total_wall_time_ms": 2262.062533999999,
      "median_handshake_ms": 319.368291,
      "median_chars_per_sec": 1148.458173368662,
      "passing_samples": 15
    },
    "comparison": {
      "ttft_difference_ms": -53.28883066666665,
      "ttft_ci_95": [
        -517.5300498466272,
        410.9523885132939
      ],
      "ttft_ci_covers_zero": true,
      "wall_time_difference_ms": -119.54819166666661,
      "wall_time_ci_95": [
        -587.1572054081456,
        348.06082207481234
      ],
      "wall_time_ci_covers_zero": true,
      "handshake_difference_ms": 30.87250573333333,
      "handshake_ci_95": [
        -62.92134681689642,
        124.66635828356307
      ],
      "handshake_ci_covers_zero": true,
      "ttft_wilcoxon_p_value": 0.5787851532947169,
      "wall_time_wilcoxon_p_value": 0.46605340600573286,
      "interpretation": "All differences within network noise at n=15. Both clients streamed successfully from Codex WS."
    }
  }
}