{
  "schema_version": 2,
  "benchmark": "safety-net-multi-backend-matrix",
  "status": "observer_residual_and_direct_run_v1",
  "backends": {
    "kiji_distilbert": {
      "pins": {
        "source_repo": "onnx-community/distilbert-NER-ONNX",
        "source_commit": "3a19fe9404a4469d91aa3d551558a97f68872f67",
        "bundle_sha256": "c129e135d86698e67c4836456212666f94a56ceaf995acd60532f557b3120d2f",
        "model_sha256": "b5f77096d0d9f425d34a2e263f8a2dfb845cdc757dc00c7a1e69e9cbb93115d5",
        "tokenizer_sha256": "cb26b43c98e8266ae3e99c2a583cf8315d73b33a17e6b20b4df7ff1f22392d34",
        "label_map_sha256": "d3753ce580a9d43b113d779c712494bd61341285317beec49cc1e848b86f9a97"
      }
    },
    "openai_privacy_filter": {
      "pins": {
        "source_repo": "openai/privacy-filter",
        "source_commit": "f7f00ca7fb869683eb732c010299d901457f19c3",
        "checkpoint_bundle_sha256": "4680158333621f3f344f58366f59612d52eff67ce6f46cff7becede5be1853ae",
        "required_opf_artifacts": [
          "config.json",
          "dtypes.json",
          "model.safetensors",
          "viterbi_calibration.json"
        ]
      }
    },
    "kiji_distilbert_int8": {
      "pins": {
        "source_repo": "onnx-community/distilbert-NER-ONNX",
        "source_commit": "3a19fe9404a4469d91aa3d551558a97f68872f67",
        "bundle_sha256": "6e7f238f38c5ee7977052ec391f6a8c68bbef038091f2ecff4747cc2268210cb",
        "model_sha256": "bd909cc924b7e10a8d2cdffb0e3bb6036f12d6ae6740cc45420fe82682f5a867",
        "tokenizer_sha256": "cb26b43c98e8266ae3e99c2a583cf8315d73b33a17e6b20b4df7ff1f22392d34",
        "label_map_sha256": "d3753ce580a9d43b113d779c712494bd61341285317beec49cc1e848b86f9a97"
      }
    }
  },
  "corpus": {
    "commit": "7348690",
    "seed": "113668362958386",
    "sha256": "c6e78cca59df550fad18e59e9877da03da82c73b80c2368e5233d76353ccfa2f",
    "coverage_report_sha256": "760f96163a68ce5f7dbc0409aa5109aa1a3ed190001536647e1881ba9d40a49c",
    "fixture_count": 150
  },
  "strict_span_leak_rate": {
    "kiji_distilbert.Global": 0.875,
    "kiji_distilbert.EnUs": 0.875,
    "kiji_distilbert.DeDe": 0.875,
    "openai_privacy_filter.Global": 0.635593,
    "openai_privacy_filter.EnUs": 0.744271,
    "openai_privacy_filter.DeDe": 0.649332,
    "kiji_distilbert_int8.Global": 0.875,
    "kiji_distilbert_int8.EnUs": 0.875,
    "kiji_distilbert_int8.DeDe": 0.875
  },
  "cells": [
    {
      "backend": "kiji_distilbert",
      "locale": "Global",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.084746,
        "recall": 0.125,
        "f1": 0.10101,
        "per_class": {
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 59,
            "true_positive": 5,
            "false_positive": 54,
            "false_negative": 0,
            "precision": 0.084746,
            "recall": 1.0,
            "f1": 0.15625
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "kiji_distilbert",
      "locale": "Global",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.084746,
        "recall": 0.5,
        "f1": 0.144928,
        "per_class": {
          "Name": {
            "support": 5,
            "predicted": 59,
            "true_positive": 5,
            "false_positive": 54,
            "false_negative": 0,
            "precision": 0.084746,
            "recall": 1.0,
            "f1": 0.15625
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "backend": "kiji_distilbert",
      "locale": "EnUs",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.54717,
        "recall": 0.125,
        "f1": 0.203509,
        "per_class": {
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 53,
            "true_positive": 29,
            "false_positive": 24,
            "false_negative": 0,
            "precision": 0.54717,
            "recall": 1.0,
            "f1": 0.707317
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "kiji_distilbert",
      "locale": "EnUs",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.54717,
        "recall": 0.5,
        "f1": 0.522523,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 53,
            "true_positive": 29,
            "false_positive": 24,
            "false_negative": 0,
            "precision": 0.54717,
            "recall": 1.0,
            "f1": 0.707317
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "backend": "kiji_distilbert",
      "locale": "DeDe",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.108614,
        "recall": 0.125,
        "f1": 0.116232,
        "per_class": {
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 267,
            "true_positive": 29,
            "false_positive": 238,
            "false_negative": 0,
            "precision": 0.108614,
            "recall": 1.0,
            "f1": 0.195946
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "kiji_distilbert",
      "locale": "DeDe",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.106227,
        "recall": 1.0,
        "f1": 0.192053,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 273,
            "true_positive": 29,
            "false_positive": 244,
            "false_negative": 0,
            "precision": 0.106227,
            "recall": 1.0,
            "f1": 0.192053
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    },
    {
      "backend": "openai_privacy_filter",
      "locale": "Global",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.321101,
        "recall": 0.364407,
        "f1": 0.341386,
        "per_class": {
          "Email": {
            "support": 59,
            "predicted": 59,
            "true_positive": 54,
            "false_positive": 5,
            "false_negative": 5,
            "precision": 0.915254,
            "recall": 0.915254,
            "f1": 0.915254
          },
          "Location": {
            "support": 0,
            "predicted": 9,
            "true_positive": 0,
            "false_positive": 9,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 7,
            "true_positive": 5,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.714286,
            "recall": 1.0,
            "f1": 0.833333
          },
          "custom:account_number": {
            "support": 0,
            "predicted": 79,
            "true_positive": 0,
            "false_positive": 79,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:date": {
            "support": 0,
            "predicted": 19,
            "true_positive": 0,
            "false_positive": 19,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 14,
            "true_positive": 11,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.785714,
            "recall": 1.0,
            "f1": 0.88
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:secret": {
            "support": 0,
            "predicted": 5,
            "true_positive": 0,
            "false_positive": 5,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:url": {
            "support": 0,
            "predicted": 26,
            "true_positive": 0,
            "false_positive": 26,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "openai_privacy_filter",
      "locale": "Global",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.035971,
        "recall": 0.5,
        "f1": 0.067114,
        "per_class": {
          "Email": {
            "support": 0,
            "predicted": 37,
            "true_positive": 0,
            "false_positive": 37,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Location": {
            "support": 0,
            "predicted": 3,
            "true_positive": 0,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:account_number": {
            "support": 0,
            "predicted": 69,
            "true_positive": 0,
            "false_positive": 69,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:date": {
            "support": 0,
            "predicted": 14,
            "true_positive": 0,
            "false_positive": 14,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:phone": {
            "support": 0,
            "predicted": 3,
            "true_positive": 0,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:secret": {
            "support": 0,
            "predicted": 6,
            "true_positive": 0,
            "false_positive": 6,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:url": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "backend": "openai_privacy_filter",
      "locale": "EnUs",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.418478,
        "recall": 0.255729,
        "f1": 0.317461,
        "per_class": {
          "Email": {
            "support": 48,
            "predicted": 48,
            "true_positive": 47,
            "false_positive": 1,
            "false_negative": 1,
            "precision": 0.979167,
            "recall": 0.979167,
            "f1": 0.979167
          },
          "Location": {
            "support": 0,
            "predicted": 11,
            "true_positive": 0,
            "false_positive": 11,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:account_number": {
            "support": 0,
            "predicted": 52,
            "true_positive": 0,
            "false_positive": 52,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:date": {
            "support": 0,
            "predicted": 11,
            "true_positive": 0,
            "false_positive": 11,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 19,
            "true_positive": 1,
            "false_positive": 18,
            "false_negative": 14,
            "precision": 0.052632,
            "recall": 0.066667,
            "f1": 0.058824
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:secret": {
            "support": 0,
            "predicted": 6,
            "true_positive": 0,
            "false_positive": 6,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:url": {
            "support": 0,
            "predicted": 8,
            "true_positive": 0,
            "false_positive": 8,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "openai_privacy_filter",
      "locale": "EnUs",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.21519,
        "recall": 0.666667,
        "f1": 0.325359,
        "per_class": {
          "Email": {
            "support": 0,
            "predicted": 31,
            "true_positive": 0,
            "false_positive": 31,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Location": {
            "support": 0,
            "predicted": 7,
            "true_positive": 0,
            "false_positive": 7,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:account_number": {
            "support": 0,
            "predicted": 63,
            "true_positive": 0,
            "false_positive": 63,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:date": {
            "support": 0,
            "predicted": 7,
            "true_positive": 0,
            "false_positive": 7,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 16,
            "true_positive": 5,
            "false_positive": 11,
            "false_negative": 10,
            "precision": 0.3125,
            "recall": 0.333333,
            "f1": 0.322581
          },
          "custom:secret": {
            "support": 0,
            "predicted": 5,
            "true_positive": 0,
            "false_positive": 5,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 0.666667,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.666667
      }
    },
    {
      "backend": "openai_privacy_filter",
      "locale": "DeDe",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.457547,
        "recall": 0.350668,
        "f1": 0.397041,
        "per_class": {
          "Email": {
            "support": 55,
            "predicted": 55,
            "true_positive": 54,
            "false_positive": 1,
            "false_negative": 1,
            "precision": 0.981818,
            "recall": 0.981818,
            "f1": 0.981818
          },
          "Location": {
            "support": 0,
            "predicted": 24,
            "true_positive": 0,
            "false_positive": 24,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:account_number": {
            "support": 0,
            "predicted": 62,
            "true_positive": 0,
            "false_positive": 62,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:date": {
            "support": 0,
            "predicted": 11,
            "true_positive": 0,
            "false_positive": 11,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 17,
            "true_positive": 14,
            "false_positive": 3,
            "false_negative": 3,
            "precision": 0.823529,
            "recall": 0.823529,
            "f1": 0.823529
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:secret": {
            "support": 0,
            "predicted": 7,
            "true_positive": 0,
            "false_positive": 7,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:url": {
            "support": 0,
            "predicted": 7,
            "true_positive": 0,
            "false_positive": 7,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "openai_privacy_filter",
      "locale": "DeDe",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.2,
        "recall": 1.0,
        "f1": 0.333333,
        "per_class": {
          "Email": {
            "support": 0,
            "predicted": 41,
            "true_positive": 0,
            "false_positive": 41,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Location": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:account_number": {
            "support": 0,
            "predicted": 55,
            "true_positive": 0,
            "false_positive": 55,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:date": {
            "support": 0,
            "predicted": 9,
            "true_positive": 0,
            "false_positive": 9,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:phone": {
            "support": 0,
            "predicted": 4,
            "true_positive": 0,
            "false_positive": 4,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:secret": {
            "support": 0,
            "predicted": 4,
            "true_positive": 0,
            "false_positive": 4,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:url": {
            "support": 0,
            "predicted": 1,
            "true_positive": 0,
            "false_positive": 1,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    },
    {
      "backend": "kiji_distilbert_int8",
      "locale": "Global",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.076923,
        "recall": 0.125,
        "f1": 0.095238,
        "per_class": {
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Location": {
            "support": 0,
            "predicted": 4,
            "true_positive": 0,
            "false_positive": 4,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 61,
            "true_positive": 5,
            "false_positive": 56,
            "false_negative": 0,
            "precision": 0.081967,
            "recall": 1.0,
            "f1": 0.151515
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "kiji_distilbert_int8",
      "locale": "EnUs",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.54717,
        "recall": 0.125,
        "f1": 0.203509,
        "per_class": {
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 53,
            "true_positive": 29,
            "false_positive": 24,
            "false_negative": 0,
            "precision": 0.54717,
            "recall": 1.0,
            "f1": 0.707317
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "kiji_distilbert_int8",
      "locale": "DeDe",
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.116,
        "recall": 0.125,
        "f1": 0.120332,
        "per_class": {
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Location": {
            "support": 0,
            "predicted": 3,
            "true_positive": 0,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 247,
            "true_positive": 29,
            "false_positive": 218,
            "false_negative": 0,
            "precision": 0.117409,
            "recall": 1.0,
            "f1": 0.210145
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "backend": "kiji_distilbert_int8",
      "locale": "Global",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.084746,
        "recall": 0.5,
        "f1": 0.144928,
        "per_class": {
          "Location": {
            "support": 0,
            "predicted": 1,
            "true_positive": 0,
            "false_positive": 1,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 58,
            "true_positive": 5,
            "false_positive": 53,
            "false_negative": 0,
            "precision": 0.086207,
            "recall": 1.0,
            "f1": 0.15873
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "backend": "kiji_distilbert_int8",
      "locale": "EnUs",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.54717,
        "recall": 0.5,
        "f1": 0.522523,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 53,
            "true_positive": 29,
            "false_positive": 24,
            "false_negative": 0,
            "precision": 0.54717,
            "recall": 1.0,
            "f1": 0.707317
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "backend": "kiji_distilbert_int8",
      "locale": "DeDe",
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.118852,
        "recall": 1.0,
        "f1": 0.212454,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 244,
            "true_positive": 29,
            "false_positive": 215,
            "false_negative": 0,
            "precision": 0.118852,
            "recall": 1.0,
            "f1": 0.212454
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    }
  ],
  "run_environment": {
    "os": "macOS-26.5-arm64-arm-64bit-Mach-O",
    "machine": "arm64",
    "python": "3.14.5",
    "onnxruntime_threads": "default",
    "precision": "int8"
  }
}
