{
  "schema_version": 1,
  "benchmark": "ner-model-leaderboard",
  "status": "real_run",
  "corpus": {
    "coverage_report_sha256": "760f96163a68ce5f7dbc0409aa5109aa1a3ed190001536647e1881ba9d40a49c",
    "sha256": "c6e78cca59df550fad18e59e9877da03da82c73b80c2368e5233d76353ccfa2f",
    "fixture_count": 150
  },
  "run_environment": {
    "os": "macOS-26.5-arm64-arm-64bit",
    "machine": "arm64",
    "python": "3.10.20",
    "warm_latency_python": "3.10.20",
    "host_note": "Apple M5 Max host; warm candidate latency measured after model load; Kiji int8 warm row measured by Rust ORT live bench."
  },
  "models": {
    "kiji-distilbert": {
      "display_name": "Kiji DistilBERT",
      "hf_repo": "onnx-community/distilbert-NER-ONNX",
      "hf_commit": "3a19fe9404a4469d91aa3d551558a97f68872f67",
      "bundle_sha256_config": "c129e135d86698e67c4836456212666f94a56ceaf995acd60532f557b3120d2f",
      "bundle_sha256_observed": "4294e3b6fa7f3529d2ec6cd2efee2499c56e9b6a16280e4d045a77fcd5921e7b",
      "license": "apache-2.0",
      "runtime": "onnxruntime",
      "wrapper": "scripts/bench/kiji-runner.py",
      "model_file": null,
      "checkpoint": "~/.cache/gaze/kiji-distilbert-3a19fe9404a4469d91aa3d551558a97f68872f67",
      "native_locales": [
        "Global"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "MISC"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "MISC": "Name"
      },
      "shippable_default": true,
      "default_caveat": null,
      "latency": {
        "cold_start_ms": 159.783041,
        "per_fixture_median_ms": 163.161312,
        "per_fixture_p95_ms": 176.925015,
        "per_fixture_p99_ms": 180.087747,
        "per_fixture_mean_ms": 165.143185,
        "throughput_bytes_per_sec": 1728.964275,
        "fixture_count": 150,
        "device": "cpu",
        "measurement_scope": "per-fixture subprocess wall clock; includes fork/import/model load because the published wrappers are one-shot CLIs"
      },
      "warm_latency": {
        "fixture": "Alice Smith visited Berlin before meeting Dr. Schmidt at Example Corp.",
        "iterations": 100,
        "warm_p50_ms": 2.562,
        "cold_start_ms": 544.124,
        "measurement_scope": "in-process ORT backend after one warmup inference"
      }
    },
    "wikineural-multilingual": {
      "display_name": "Babelscape WikiNeural Multilingual",
      "hf_repo": "Babelscape/wikineural-multilingual-ner",
      "hf_commit": "bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1",
      "bundle_sha256_config": "<unpinned>",
      "bundle_sha256_observed": "7015eaabce63ccc6ed759eeb517c9643f70a098aa49c57131d47aaf36aee075a",
      "license": "cc-by-nc-sa-4.0",
      "runtime": "transformers",
      "wrapper": "scripts/bench/transformers-runner.py",
      "checkpoint": "~/.anvil/worktrees/gaze-empiretwo/v0.9-multi-ner-bench/.huggingface-cache/models/wikineural-multilingual",
      "native_locales": [
        "EnUs",
        "DeDe",
        "FrFr",
        "ItIt",
        "EsEs",
        "Global"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "MISC"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "MISC": "Custom:other"
      },
      "shippable_default": false,
      "default_caveat": "Non-commercial license; informational benchmark only.",
      "latency": {
        "cold_start_ms": 2890.127959,
        "per_fixture_median_ms": 1786.36075,
        "per_fixture_p95_ms": 1997.942931,
        "per_fixture_p99_ms": 2614.783255,
        "per_fixture_mean_ms": 1832.978229,
        "throughput_bytes_per_sec": 155.77199,
        "fixture_count": 150,
        "device": "cpu",
        "measurement_scope": "per-fixture subprocess wall clock; includes fork/import/model load because the published wrappers are one-shot CLIs"
      }
    },
    "dslim-bert-base-ner": {
      "display_name": "dslim/bert-base-NER (English)",
      "hf_repo": "dslim/bert-base-NER",
      "hf_commit": "d1a3e8f13f8c3566299d95fcfc9a8d2382a9affc",
      "bundle_sha256_config": "<unpinned>",
      "bundle_sha256_observed": "fd7119ab25b127f50b4165bde3cc2466c90d0eabe230b206bf3dae7b6225fe54",
      "license": "mit",
      "runtime": "transformers",
      "wrapper": "scripts/bench/transformers-runner.py",
      "checkpoint": "~/.anvil/worktrees/gaze-empiretwo/v0.9-multi-ner-bench/.huggingface-cache/models/dslim-bert-base-ner",
      "native_locales": [
        "EnUs",
        "EnGb"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "MISC"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "MISC": "Custom:other"
      },
      "shippable_default": true,
      "default_caveat": null,
      "latency": {
        "cold_start_ms": 1783.4385,
        "per_fixture_median_ms": 1737.297417,
        "per_fixture_p95_ms": 1808.988923,
        "per_fixture_p99_ms": 1918.146869,
        "per_fixture_mean_ms": 1743.286581,
        "throughput_bytes_per_sec": 163.786419,
        "fixture_count": 150,
        "device": "cpu",
        "measurement_scope": "per-fixture subprocess wall clock; includes fork/import/model load because the published wrappers are one-shot CLIs"
      }
    },
    "davlan-bert-multilingual": {
      "display_name": "Davlan multilingual BERT-NER (HRL)",
      "hf_repo": "Davlan/bert-base-multilingual-cased-ner-hrl",
      "hf_commit": "e756de7f7b8f64fea0c3d7c3872f1322fab747b1",
      "bundle_sha256_config": "<unpinned>",
      "bundle_sha256_observed": "c3af1f92b9292ba894177812d6f89e5d93502ef819b7fadf7ec7aeb692b4c49d",
      "license": "afl-3.0",
      "runtime": "transformers",
      "wrapper": "scripts/bench/transformers-runner.py",
      "checkpoint": "~/.anvil/worktrees/gaze-empiretwo/v0.9-multi-ner-bench/.huggingface-cache/models/davlan-bert-multilingual",
      "native_locales": [
        "EnUs",
        "DeDe",
        "FrFr",
        "ItIt",
        "EsEs",
        "PtBr",
        "Global"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "DATE"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "DATE": "Custom:date"
      },
      "shippable_default": true,
      "default_caveat": null,
      "latency": {
        "cold_start_ms": 2637.689166,
        "per_fixture_median_ms": 1911.348979,
        "per_fixture_p95_ms": 2143.359539,
        "per_fixture_p99_ms": 2295.070171,
        "per_fixture_mean_ms": 1944.522838,
        "throughput_bytes_per_sec": 146.836366,
        "fixture_count": 150,
        "device": "cpu",
        "measurement_scope": "per-fixture subprocess wall clock; includes fork/import/model load because the published wrappers are one-shot CLIs"
      }
    },
    "openobscure-tinybert4l-pii-ner-int8": {
      "display_name": "openobscure TinyBERT4L PII NER int8",
      "hf_repo": "openobscure/tinybert4l-pii-ner-int8",
      "hf_commit": "f8399a9431b0202e78e20b2d38a3385521472f3f",
      "bundle_sha256_config": "f5fa446a02c7d71251749a37c31b5f8ae3c450360b97e3df99539d688ff43d9a",
      "bundle_sha256_observed": "f5fa446a02c7d71251749a37c31b5f8ae3c450360b97e3df99539d688ff43d9a",
      "license": "apache-2.0",
      "runtime": "onnxruntime",
      "wrapper": "scripts/bench/onnx-token-classification-runner.py",
      "model_file": "model_int8.onnx",
      "checkpoint": ".huggingface-cache/models/openobscure-tinybert4l-pii-ner-int8",
      "native_locales": [
        "Global"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "HEALTH",
        "CHILD"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "HEALTH": "Custom:health",
        "CHILD": "Custom:child"
      },
      "shippable_default": true,
      "default_caveat": "PII-specific TinyBERT head; not a strict CoNLL label-set match.",
      "latency": {
        "cold_start_ms": 75.117042,
        "per_fixture_median_ms": 76.547895,
        "per_fixture_p95_ms": 78.320183,
        "per_fixture_p99_ms": 78.746177,
        "per_fixture_mean_ms": 76.617194,
        "throughput_bytes_per_sec": 3726.665662,
        "fixture_count": 150,
        "device": "cpu",
        "measurement_scope": "per-fixture subprocess wall clock; includes fork/import/model load because the published wrappers are one-shot CLIs"
      },
      "warm_latency": {
        "fixture_count": 150,
        "warm_p50_ms": 1.086937,
        "warm_p95_ms": 2.220792,
        "warm_mean_ms": 1.31441,
        "measurement_scope": "persistent Python process with model/session loaded once; direct corpus texts"
      }
    },
    "mrm8488-mobilebert-ner": {
      "display_name": "mrm8488 MobileBERT NER",
      "hf_repo": "mrm8488/mobilebert-finetuned-ner",
      "hf_commit": "3f9a1f308e4272219bd43e9127cb65fa5504d9ec",
      "bundle_sha256_config": "2c867d5c7117ef864d82dc2b47089c45550ddabf4c24bd9dfaa6e96026e6f19a",
      "bundle_sha256_observed": "2c867d5c7117ef864d82dc2b47089c45550ddabf4c24bd9dfaa6e96026e6f19a",
      "license": "mit",
      "runtime": "transformers",
      "wrapper": "scripts/bench/transformers-runner.py",
      "model_file": null,
      "checkpoint": ".huggingface-cache/models/mrm8488-mobilebert-ner",
      "native_locales": [
        "EnUs",
        "EnGb"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "MISC"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "MISC": "Custom:other"
      },
      "shippable_default": true,
      "default_caveat": null,
      "latency": {
        "cold_start_ms": 1782.920125,
        "per_fixture_median_ms": 1760.778375,
        "per_fixture_p95_ms": 1778.697129,
        "per_fixture_p99_ms": 1783.6371,
        "per_fixture_mean_ms": 1761.919148,
        "throughput_bytes_per_sec": 162.054353,
        "fixture_count": 150,
        "device": "cpu",
        "measurement_scope": "per-fixture subprocess wall clock; includes fork/import/model load because the published wrappers are one-shot CLIs"
      },
      "warm_latency": {
        "fixture_count": 150,
        "warm_p50_ms": 16.493916,
        "warm_p95_ms": 21.804541,
        "warm_mean_ms": 15.030467,
        "measurement_scope": "persistent Python process with Transformers pipeline loaded once; direct corpus texts"
      }
    },
    "osiria-minilm-italian-ner": {
      "display_name": "osiria MiniLM-L6-H384 Italian NER",
      "hf_repo": "osiria/minilm-l6-h384-italian-cased-ner",
      "hf_commit": "125c646df9361d4bc764a6b1710bee21b50cd924",
      "bundle_sha256_config": "46c82257388b9727b752679cc13d4594fb9eae19dfbf2d2f225f4d5bae841f0a",
      "bundle_sha256_observed": "46c82257388b9727b752679cc13d4594fb9eae19dfbf2d2f225f4d5bae841f0a",
      "license": "mit",
      "runtime": "transformers",
      "wrapper": "scripts/bench/transformers-runner.py",
      "model_file": null,
      "checkpoint": ".huggingface-cache/models/osiria-minilm-italian-ner",
      "native_locales": [
        "ItIt"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "MISC"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "MISC": "Custom:other"
      },
      "shippable_default": true,
      "default_caveat": "MiniLM-family NER head is Italian-native; no permissive English/Multi MiniLM NER head was found.",
      "latency": {
        "cold_start_ms": 1564.524125,
        "per_fixture_median_ms": 1566.990562,
        "per_fixture_p95_ms": 1583.862771,
        "per_fixture_p99_ms": 1639.894385,
        "per_fixture_mean_ms": 1570.615786,
        "throughput_bytes_per_sec": 181.792816,
        "fixture_count": 150,
        "device": "cpu",
        "measurement_scope": "per-fixture subprocess wall clock; includes fork/import/model load because the published wrappers are one-shot CLIs"
      },
      "warm_latency": {
        "fixture_count": 150,
        "warm_p50_ms": 4.702688,
        "warm_p95_ms": 9.075,
        "warm_mean_ms": 5.77294,
        "measurement_scope": "persistent Python process with Transformers pipeline loaded once; direct corpus texts"
      }
    },
    "kiji-distilbert-int8-ort": {
      "display_name": "Kiji DistilBERT int8 ORT",
      "hf_repo": "onnx-community/distilbert-NER-ONNX",
      "hf_commit": "3a19fe9404a4469d91aa3d551558a97f68872f67",
      "bundle_sha256_config": "6e7f238f38c5ee7977052ec391f6a8c68bbef038091f2ecff4747cc2268210cb",
      "license": "apache-2.0",
      "runtime": "onnxruntime-rust",
      "wrapper": null,
      "native_locales": [
        "Global"
      ],
      "label_set": [
        "PER",
        "LOC",
        "ORG",
        "MISC"
      ],
      "label_map": {
        "PER": "Name",
        "LOC": "Location",
        "ORG": "Organization",
        "MISC": "Name"
      },
      "shippable_default": true,
      "default_caveat": "Tier 2 int8 ORT comparison row; recall cells are identical to Kiji DistilBERT by gate.",
      "warm_latency": {
        "fixture": "Alice Smith visited Berlin before meeting Dr. Schmidt at Example Corp.",
        "iterations": 100,
        "warm_p50_ms": 1.849,
        "cold_start_ms": 192.613,
        "measurement_scope": "in-process ORT backend after one warmup inference"
      }
    }
  },
  "cells": [
    {
      "model": "wikineural-multilingual",
      "locale": "Global",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.3125,
        "recall": 0.125,
        "f1": 0.178571,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 8,
            "true_positive": 0,
            "false_positive": 8,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 3,
            "true_positive": 0,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "wikineural-multilingual",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.707317,
        "recall": 0.125,
        "f1": 0.212454,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 9,
            "true_positive": 0,
            "false_positive": 9,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 3,
            "true_positive": 0,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "wikineural-multilingual",
      "locale": "DeDe",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.408451,
        "recall": 0.125,
        "f1": 0.191419,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 27,
            "true_positive": 0,
            "false_positive": 27,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 42,
            "true_positive": 29,
            "false_positive": 13,
            "false_negative": 0,
            "precision": 0.690476,
            "recall": 1.0,
            "f1": 0.816901
          },
          "Organization": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "wikineural-multilingual",
      "locale": "Global",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.277778,
        "recall": 0.5,
        "f1": 0.357143,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 12,
            "true_positive": 0,
            "false_positive": 12,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 1,
            "true_positive": 0,
            "false_positive": 1,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "wikineural-multilingual",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.707317,
        "recall": 0.5,
        "f1": 0.585859,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 12,
            "true_positive": 0,
            "false_positive": 12,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "wikineural-multilingual",
      "locale": "DeDe",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.537037,
        "recall": 1.0,
        "f1": 0.698795,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 23,
            "true_positive": 0,
            "false_positive": 23,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    },
    {
      "model": "dslim-bert-base-ner",
      "locale": "Global",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.102041,
        "recall": 0.125,
        "f1": 0.11236,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 15,
            "true_positive": 0,
            "false_positive": 15,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 29,
            "true_positive": 0,
            "false_positive": 29,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "dslim-bert-base-ner",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.636364,
        "recall": 0.12069,
        "f1": 0.202899,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 3,
            "true_positive": 0,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 28,
            "true_positive": 28,
            "false_positive": 0,
            "false_negative": 1,
            "precision": 1.0,
            "recall": 0.965517,
            "f1": 0.982456
          },
          "Organization": {
            "support": 0,
            "predicted": 13,
            "true_positive": 0,
            "false_positive": 13,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "dslim-bert-base-ner",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.162921,
        "recall": 0.125,
        "f1": 0.141463,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 12,
            "true_positive": 0,
            "false_positive": 12,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 30,
            "true_positive": 29,
            "false_positive": 1,
            "false_negative": 0,
            "precision": 0.966667,
            "recall": 1.0,
            "f1": 0.983051
          },
          "Organization": {
            "support": 0,
            "predicted": 136,
            "true_positive": 0,
            "false_positive": 136,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "dslim-bert-base-ner",
      "locale": "Global",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.104167,
        "recall": 0.5,
        "f1": 0.172414,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 8,
            "true_positive": 0,
            "false_positive": 8,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 35,
            "true_positive": 0,
            "false_positive": 35,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "dslim-bert-base-ner",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.613636,
        "recall": 0.465517,
        "f1": 0.529412,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 27,
            "true_positive": 27,
            "false_positive": 0,
            "false_negative": 2,
            "precision": 1.0,
            "recall": 0.931034,
            "f1": 0.964286
          },
          "Organization": {
            "support": 0,
            "predicted": 17,
            "true_positive": 0,
            "false_positive": 17,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.465517,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.465517
      }
    },
    {
      "model": "dslim-bert-base-ner",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.153439,
        "recall": 1.0,
        "f1": 0.266055,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 5,
            "true_positive": 0,
            "false_positive": 5,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 155,
            "true_positive": 0,
            "false_positive": 155,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    },
    {
      "model": "davlan-bert-multilingual",
      "locale": "Global",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.294118,
        "recall": 0.125,
        "f1": 0.175439,
        "per_class": {
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 12,
            "true_positive": 0,
            "false_positive": 12,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "davlan-bert-multilingual",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.690476,
        "recall": 0.125,
        "f1": 0.211679,
        "per_class": {
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 13,
            "true_positive": 0,
            "false_positive": 13,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "davlan-bert-multilingual",
      "locale": "DeDe",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.690476,
        "recall": 0.125,
        "f1": 0.211679,
        "per_class": {
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 13,
            "true_positive": 0,
            "false_positive": 13,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "davlan-bert-multilingual",
      "locale": "Global",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.294118,
        "recall": 0.5,
        "f1": 0.37037,
        "per_class": {
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 12,
            "true_positive": 0,
            "false_positive": 12,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "davlan-bert-multilingual",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.666667,
        "recall": 0.482758,
        "f1": 0.56,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 28,
            "true_positive": 28,
            "false_positive": 0,
            "false_negative": 1,
            "precision": 1.0,
            "recall": 0.965517,
            "f1": 0.982456
          },
          "Organization": {
            "support": 0,
            "predicted": 14,
            "true_positive": 0,
            "false_positive": 14,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.482758,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.482758
      }
    },
    {
      "model": "davlan-bert-multilingual",
      "locale": "DeDe",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.591837,
        "recall": 1.0,
        "f1": 0.74359,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 20,
            "true_positive": 0,
            "false_positive": 20,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    },
    {
      "model": "kiji-distilbert",
      "locale": "Global",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.084746,
        "recall": 0.125,
        "f1": 0.10101,
        "per_class": {
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 59,
            "true_positive": 5,
            "false_positive": 54,
            "false_negative": 0,
            "precision": 0.084746,
            "recall": 1.0,
            "f1": 0.15625
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "kiji-distilbert",
      "locale": "EnUs",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.54717,
        "recall": 0.125,
        "f1": 0.203509,
        "per_class": {
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 53,
            "true_positive": 29,
            "false_positive": 24,
            "false_negative": 0,
            "precision": 0.54717,
            "recall": 1.0,
            "f1": 0.707317
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "kiji-distilbert",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.108614,
        "recall": 0.125,
        "f1": 0.116232,
        "per_class": {
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 267,
            "true_positive": 29,
            "false_positive": 238,
            "false_negative": 0,
            "precision": 0.108614,
            "recall": 1.0,
            "f1": 0.195946
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "kiji-distilbert",
      "locale": "Global",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.084746,
        "recall": 0.5,
        "f1": 0.144928,
        "per_class": {
          "Location": {
            "support": 0,
            "predicted": 1,
            "true_positive": 0,
            "false_positive": 1,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 58,
            "true_positive": 5,
            "false_positive": 53,
            "false_negative": 0,
            "precision": 0.086207,
            "recall": 1.0,
            "f1": 0.15873
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "kiji-distilbert",
      "locale": "EnUs",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.54717,
        "recall": 0.5,
        "f1": 0.522523,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 53,
            "true_positive": 29,
            "false_positive": 24,
            "false_negative": 0,
            "precision": 0.54717,
            "recall": 1.0,
            "f1": 0.707317
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "kiji-distilbert",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.107011,
        "recall": 1.0,
        "f1": 0.193333,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 271,
            "true_positive": 29,
            "false_positive": 242,
            "false_negative": 0,
            "precision": 0.107011,
            "recall": 1.0,
            "f1": 0.193333
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    },
    {
      "model": "openobscure-tinybert4l-pii-ner-int8",
      "locale": "Global",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.428571,
        "recall": 0.075,
        "f1": 0.12766,
        "per_class": {
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 3,
            "true_positive": 3,
            "false_positive": 0,
            "false_negative": 2,
            "precision": 1.0,
            "recall": 0.6,
            "f1": 0.75
          },
          "Organization": {
            "support": 0,
            "predicted": 4,
            "true_positive": 0,
            "false_positive": 4,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "openobscure-tinybert4l-pii-ner-int8",
      "locale": "EnUs",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.73913,
        "recall": 0.073276,
        "f1": 0.133333,
        "per_class": {
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 18,
            "true_positive": 17,
            "false_positive": 1,
            "false_negative": 12,
            "precision": 0.944444,
            "recall": 0.586207,
            "f1": 0.723404
          },
          "Organization": {
            "support": 0,
            "predicted": 5,
            "true_positive": 0,
            "false_positive": 5,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "openobscure-tinybert4l-pii-ner-int8",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.105263,
        "recall": 0.060345,
        "f1": 0.076712,
        "per_class": {
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 16,
            "true_positive": 14,
            "false_positive": 2,
            "false_negative": 15,
            "precision": 0.875,
            "recall": 0.482759,
            "f1": 0.622222
          },
          "Organization": {
            "support": 0,
            "predicted": 117,
            "true_positive": 0,
            "false_positive": 117,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "openobscure-tinybert4l-pii-ner-int8",
      "locale": "Global",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.5,
        "recall": 0.4,
        "f1": 0.444444,
        "per_class": {
          "Name": {
            "support": 5,
            "predicted": 4,
            "true_positive": 4,
            "false_positive": 0,
            "false_negative": 1,
            "precision": 1.0,
            "recall": 0.8,
            "f1": 0.888889
          },
          "Organization": {
            "support": 0,
            "predicted": 4,
            "true_positive": 0,
            "false_positive": 4,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.4,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.4
      }
    },
    {
      "model": "openobscure-tinybert4l-pii-ner-int8",
      "locale": "EnUs",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.857143,
        "recall": 0.206897,
        "f1": 0.333333,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 12,
            "true_positive": 12,
            "false_positive": 0,
            "false_negative": 17,
            "precision": 1.0,
            "recall": 0.413793,
            "f1": 0.585366
          },
          "Organization": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.206897,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.206897
      }
    },
    {
      "model": "openobscure-tinybert4l-pii-ner-int8",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.075949,
        "recall": 0.206897,
        "f1": 0.111111,
        "per_class": {
          "Name": {
            "support": 29,
            "predicted": 7,
            "true_positive": 6,
            "false_positive": 1,
            "false_negative": 23,
            "precision": 0.857143,
            "recall": 0.206897,
            "f1": 0.333333
          },
          "Organization": {
            "support": 0,
            "predicted": 72,
            "true_positive": 0,
            "false_positive": 72,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 0.206897,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.206897
      }
    },
    {
      "model": "mrm8488-mobilebert-ner",
      "locale": "Global",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.138889,
        "recall": 0.125,
        "f1": 0.131579,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 26,
            "true_positive": 0,
            "false_positive": 26,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 8,
            "true_positive": 5,
            "false_positive": 3,
            "false_negative": 0,
            "precision": 0.625,
            "recall": 1.0,
            "f1": 0.769231
          },
          "Organization": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "mrm8488-mobilebert-ner",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.474576,
        "recall": 0.12069,
        "f1": 0.19244,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 14,
            "true_positive": 0,
            "false_positive": 14,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 44,
            "true_positive": 28,
            "false_positive": 16,
            "false_negative": 1,
            "precision": 0.636364,
            "recall": 0.965517,
            "f1": 0.767123
          },
          "Organization": {
            "support": 0,
            "predicted": 1,
            "true_positive": 0,
            "false_positive": 1,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "mrm8488-mobilebert-ner",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.083815,
        "recall": 0.125,
        "f1": 0.100346,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 75,
            "true_positive": 0,
            "false_positive": 75,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Location": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 47,
            "true_positive": 29,
            "false_positive": 18,
            "false_negative": 0,
            "precision": 0.617021,
            "recall": 1.0,
            "f1": 0.763158
          },
          "Organization": {
            "support": 0,
            "predicted": 222,
            "true_positive": 0,
            "false_positive": 222,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "mrm8488-mobilebert-ner",
      "locale": "Global",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.294118,
        "recall": 0.5,
        "f1": 0.37037,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 12,
            "true_positive": 0,
            "false_positive": 12,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "mrm8488-mobilebert-ner",
      "locale": "EnUs",
      "native_locale": true,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.7,
        "recall": 0.482758,
        "f1": 0.571428,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 11,
            "true_positive": 0,
            "false_positive": 11,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 28,
            "true_positive": 28,
            "false_positive": 0,
            "false_negative": 1,
            "precision": 1.0,
            "recall": 0.965517,
            "f1": 0.982456
          },
          "Organization": {
            "support": 0,
            "predicted": 1,
            "true_positive": 0,
            "false_positive": 1,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.482758,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.482758
      }
    },
    {
      "model": "mrm8488-mobilebert-ner",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.090909,
        "recall": 1.0,
        "f1": 0.166667,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 79,
            "true_positive": 0,
            "false_positive": 79,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 35,
            "true_positive": 29,
            "false_positive": 6,
            "false_negative": 0,
            "precision": 0.828571,
            "recall": 1.0,
            "f1": 0.90625
          },
          "Organization": {
            "support": 0,
            "predicted": 205,
            "true_positive": 0,
            "false_positive": 205,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    },
    {
      "model": "osiria-minilm-italian-ner",
      "locale": "Global",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.02551,
        "recall": 0.125,
        "f1": 0.042373,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 125,
            "true_positive": 0,
            "false_positive": 125,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 59,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 59,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 66,
            "true_positive": 0,
            "false_positive": 66,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 21,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 21,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 18,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 18,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 25,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 25,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 11,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 11,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "osiria-minilm-italian-ner",
      "locale": "EnUs",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.215385,
        "recall": 0.12069,
        "f1": 0.154696,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 39,
            "true_positive": 0,
            "false_positive": 39,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 48,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 48,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 28,
            "true_positive": 28,
            "false_positive": 0,
            "false_negative": 1,
            "precision": 1.0,
            "recall": 0.965517,
            "f1": 0.982456
          },
          "Organization": {
            "support": 0,
            "predicted": 63,
            "true_positive": 0,
            "false_positive": 63,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 5,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 5,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 8,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 8,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 10,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 10,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "osiria-minilm-italian-ner",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "direct_detector",
      "metrics": {
        "precision": 0.111111,
        "recall": 0.125,
        "f1": 0.117647,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 104,
            "true_positive": 0,
            "false_positive": 104,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Email": {
            "support": 55,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 55,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 128,
            "true_positive": 0,
            "false_positive": 128,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:credit_card": {
            "support": 7,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 7,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:eth_address": {
            "support": 4,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 4,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:iban": {
            "support": 22,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 22,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:ip_address": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:phone": {
            "support": 17,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 17,
            "precision": null,
            "recall": 0.0,
            "f1": null
          },
          "custom:postal_code": {
            "support": 13,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 13,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        }
      }
    },
    {
      "model": "osiria-minilm-italian-ner",
      "locale": "Global",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.041322,
        "recall": 0.5,
        "f1": 0.076336,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 114,
            "true_positive": 0,
            "false_positive": 114,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 5,
            "predicted": 5,
            "true_positive": 5,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 2,
            "true_positive": 0,
            "false_positive": 2,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "custom:postal_code": {
            "support": 6,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 6,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "osiria-minilm-italian-ner",
      "locale": "EnUs",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.432836,
        "recall": 0.5,
        "f1": 0.464,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 38,
            "true_positive": 0,
            "false_positive": 38,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "custom:phone": {
            "support": 15,
            "predicted": 0,
            "true_positive": 0,
            "false_positive": 0,
            "false_negative": 15,
            "precision": null,
            "recall": 0.0,
            "f1": null
          }
        },
        "observer_residual_recall": 0.5,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 0.5
      }
    },
    {
      "model": "osiria-minilm-italian-ner",
      "locale": "DeDe",
      "native_locale": false,
      "mode": "observer_residual",
      "metrics": {
        "precision": 0.143564,
        "recall": 1.0,
        "f1": 0.251082,
        "per_class": {
          "Custom:other": {
            "support": 0,
            "predicted": 123,
            "true_positive": 0,
            "false_positive": 123,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          },
          "Name": {
            "support": 29,
            "predicted": 29,
            "true_positive": 29,
            "false_positive": 0,
            "false_negative": 0,
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0
          },
          "Organization": {
            "support": 0,
            "predicted": 50,
            "true_positive": 0,
            "false_positive": 50,
            "false_negative": 0,
            "precision": 0.0,
            "recall": null,
            "f1": null
          }
        },
        "observer_residual_recall": 1.0,
        "agreement_with_rule_floor": null,
        "expansion_fraction": null,
        "contradiction_fraction": null,
        "novel_tp_over_rule_floor": 1.0
      }
    }
  ]
}
