{
  "id": "same",
  "title": "Same person",
  "question": "Is this profile the same real person as this CRM record?",
  "plain_question": "Is this profile the same person as this CRM record?",
  "matters": "A wrong match attaches the wrong person's data to your lead.",
  "n": 200,
  "primary_metric": {
    "name": "Correct",
    "definition": "Share of the 200 records where the model's first answer to \"same real person as this CRM record?\" matched the answer we already knew. A failed call counts as wrong."
  },
  "guard_definition": "Share of records the model can settle by itself with at most 1 mistake in 100, at the confidence cutoff that works best for it. Only a \"yes\" counts, because a wrong profile match costs the most.",
  "headline": "The best model, jev-1.14, gets 88% of profile matches right and the next four are too close to call. A free rule built from the answer key gets 90%.",
  "leader": "typesafe/jev-1.14",
  "rows": [
    {
      "model": "company, email label or exact title",
      "provider": null,
      "kind": "rule fitted on this key",
      "metric": 0.9,
      "ci_low": 0.85,
      "ci_high": 0.93,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": null,
      "usd_per_1k_cases": 0,
      "median_latency_s": null,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Free rule built by looking at this answer key, so its score runs high. Not tested against the leader."
    },
    {
      "model": "typesafe/jev-1.14",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.88,
      "ci_low": 0.83,
      "ci_high": 0.92,
      "guard": {
        "share": 0.02,
        "threshold": 0.93,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0277,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Leader."
    },
    {
      "model": "perplexity/pplx-decider-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.88,
      "ci_low": 0.82,
      "ci_high": 0.91,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": 0,
      "usd_per_1k_cases": 0.0077,
      "median_latency_s": 0.68,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "liquid/d1-ultra",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.87,
      "ci_low": 0.82,
      "ci_high": 0.91,
      "guard": {
        "share": 0.02,
        "threshold": 0.98,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.08,
      "stability": 0,
      "usd_per_1k_cases": 0.0163,
      "median_latency_s": 0.63,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "openai/gpt-6-luna",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.87,
      "ci_low": 0.82,
      "ci_high": 0.91,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0.06,
      "usd_per_1k_cases": 0.051,
      "median_latency_s": 1.43,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "liquid/d1.1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.86,
      "ci_low": 0.81,
      "ci_high": 0.91,
      "guard": {
        "share": 0.07,
        "threshold": 0.99,
        "passed": 13,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0077,
      "median_latency_s": 0.61,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "typesafe/jev-1.13",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.86,
      "ci_low": 0.81,
      "ci_high": 0.91,
      "guard": {
        "share": 0.04,
        "threshold": 0.92,
        "passed": 7,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0277,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "nace-ai/drex-max",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.86,
      "ci_low": 0.81,
      "ci_high": 0.91,
      "guard": {
        "share": 0.03,
        "threshold": 1,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0233,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "google/gemini-4-argon",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.86,
      "ci_low": 0.81,
      "ci_high": 0.91,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0,
      "usd_per_1k_cases": 1.5273,
      "median_latency_s": 4.05,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "liquid/d1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.86,
      "ci_low": 0.81,
      "ci_high": 0.9,
      "guard": {
        "share": 0.03,
        "threshold": 1,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0077,
      "median_latency_s": 0.62,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "google/gemini-3.8-flash",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.86,
      "ci_low": 0.81,
      "ci_high": 0.9,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0.01,
      "usd_per_1k_cases": 0.4497,
      "median_latency_s": 2.04,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "the record's exact title among the profile's role titles",
      "provider": null,
      "kind": "rule fitted on this key",
      "metric": 0.86,
      "ci_low": 0.81,
      "ci_high": 0.9,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.14,
      "stability": null,
      "usd_per_1k_cases": 0,
      "median_latency_s": null,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Free rule built by looking at this answer key, so its score runs high. Not tested against the leader."
    },
    {
      "model": "openai/gpt-6-luna-decisions",
      "provider": "openai",
      "kind": "decision model",
      "metric": 0.85,
      "ci_low": 0.8,
      "ci_high": 0.9,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.09,
      "stability": 0,
      "usd_per_1k_cases": 0.0383,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "perplexity/pplx-decider-flash-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.85,
      "ci_low": 0.8,
      "ci_high": 0.9,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0,
      "usd_per_1k_cases": 0.004,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "baseten/mammoth-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.85,
      "ci_low": 0.79,
      "ci_high": 0.89,
      "guard": {
        "share": 0.04,
        "threshold": 1,
        "passed": 7,
        "wrong": 0
      },
      "ece": 0.09,
      "stability": 0,
      "usd_per_1k_cases": 0.0407,
      "median_latency_s": 0.61,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "morph/reflex",
      "provider": "morph",
      "kind": "decision model",
      "metric": 0.85,
      "ci_low": 0.79,
      "ci_high": 0.89,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 3,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0167,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "microsoft/microsoft-decision-1",
      "provider": "microsoft",
      "kind": "decision model",
      "metric": 0.85,
      "ci_low": 0.79,
      "ci_high": 0.89,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.07,
      "stability": 0,
      "usd_per_1k_cases": 0.0113,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "inception/mercury-decide",
      "provider": "inception",
      "kind": "decision model",
      "metric": 0.85,
      "ci_low": 0.79,
      "ci_high": 0.89,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": 0.03,
      "usd_per_1k_cases": 0.007,
      "median_latency_s": 0.69,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "openai/gpt-6.1-terra",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.85,
      "ci_low": 0.79,
      "ci_high": 0.89,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0.04,
      "usd_per_1k_cases": 0.5113,
      "median_latency_s": 1.47,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "anthropic/claude-haiku-5.5",
      "provider": "anthropic",
      "kind": "LLM",
      "metric": 0.84,
      "ci_low": 0.78,
      "ci_high": 0.88,
      "guard": {
        "share": 0.02,
        "threshold": 0.97,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0857,
      "median_latency_s": 0.93,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "deepseek/deepseek-v4.1-flash-1017",
      "provider": "deepseek",
      "kind": "LLM",
      "metric": 0.84,
      "ci_low": 0.78,
      "ci_high": 0.88,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0.01,
      "usd_per_1k_cases": 0.027,
      "median_latency_s": 0.97,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "z-ai/glm-5.3-flash",
      "provider": "z-ai",
      "kind": "LLM",
      "metric": 0.83,
      "ci_low": 0.78,
      "ci_high": 0.88,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0.09,
      "usd_per_1k_cases": 0.0397,
      "median_latency_s": 1.1,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "cloudflare/clef-flash",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.83,
      "ci_low": 0.77,
      "ci_high": 0.88,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.14,
      "stability": 0,
      "usd_per_1k_cases": 0.0103,
      "median_latency_s": 0.74,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "baseten/elephant-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.82,
      "ci_low": 0.77,
      "ci_high": 0.87,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0,
      "usd_per_1k_cases": 0.0143,
      "median_latency_s": 0.61,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "minimax/minimax-m3.1-flash",
      "provider": "minimax",
      "kind": "LLM",
      "metric": 0.82,
      "ci_low": 0.77,
      "ci_high": 0.87,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": 0,
      "usd_per_1k_cases": 0.0563,
      "median_latency_s": 0.9,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "cloudflare/clef",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.81,
      "ci_low": 0.74,
      "ci_high": 0.85,
      "guard": {
        "share": 0.04,
        "threshold": 0.99,
        "passed": 7,
        "wrong": 0
      },
      "ece": 0.18,
      "stability": 0.03,
      "usd_per_1k_cases": 0.0853,
      "median_latency_s": 0.8,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "jaredpalmer/kev-4b",
      "provider": "jaredpalmer",
      "kind": "decision model",
      "metric": 0.8,
      "ci_low": 0.74,
      "ci_high": 0.85,
      "guard": {
        "share": 0.04,
        "threshold": 0.96,
        "passed": 8,
        "wrong": 0
      },
      "ece": 0.17,
      "stability": 0,
      "usd_per_1k_cases": 0.0123,
      "median_latency_s": 0.65,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "cloudflare/clef-omni",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.8,
      "ci_low": 0.74,
      "ci_high": 0.85,
      "guard": {
        "share": 0.02,
        "threshold": 0.99,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0,
      "usd_per_1k_cases": 0.058,
      "median_latency_s": 0.65,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "nace-ai/drex-v2.0",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.79,
      "ci_low": 0.72,
      "ci_high": 0.84,
      "guard": {
        "share": 0.03,
        "threshold": 0.95,
        "passed": 5,
        "wrong": 0
      },
      "ece": 0.14,
      "stability": 0,
      "usd_per_1k_cases": 0.0173,
      "median_latency_s": 0.8,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "parallel/fast",
      "provider": "parallel",
      "kind": "web agent",
      "metric": 0.73,
      "ci_low": 0.67,
      "ci_high": 0.79,
      "guard": {
        "share": 0.01,
        "threshold": 0.98,
        "passed": 3,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0.04,
      "usd_per_1k_cases": 1,
      "median_latency_s": 0.94,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "folded name and the record's company among the profile's roles",
      "provider": null,
      "kind": "baseline",
      "metric": 0.71,
      "ci_low": 0.65,
      "ci_high": 0.77,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.29,
      "stability": null,
      "usd_per_1k_cases": 0,
      "median_latency_s": null,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Free comparison rule with no model. Not tested against the leader."
    },
    {
      "model": "togethercomputer/tev1-4b-experimental",
      "provider": "togethercomputer",
      "kind": "decision model",
      "metric": 0.68,
      "ci_low": 0.61,
      "ci_high": 0.74,
      "guard": {
        "share": 0.07,
        "threshold": 0.89,
        "passed": 13,
        "wrong": 0
      },
      "ece": 0.22,
      "stability": 0,
      "usd_per_1k_cases": 0.014,
      "median_latency_s": 0.65,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "scaledown/classify-1",
      "provider": "scaledown",
      "kind": "decision model",
      "metric": 0.67,
      "ci_low": 0.6,
      "ci_high": 0.73,
      "guard": {
        "share": 0.04,
        "threshold": 0.85,
        "passed": 8,
        "wrong": 0
      },
      "ece": 0.2,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0193,
      "median_latency_s": 0.72,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "upstage/solar-decide",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.65,
      "ci_low": 0.58,
      "ci_high": 0.71,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.29,
      "stability": 0.14,
      "usd_per_1k_cases": 0.031,
      "median_latency_s": 1.39,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "baseten/joey-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.62,
      "ci_low": 0.55,
      "ci_high": 0.68,
      "guard": {
        "share": 0.01,
        "threshold": 0.86,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0,
      "usd_per_1k_cases": 0.004,
      "median_latency_s": 0.61,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "upstage/solar-decide-flash",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.58,
      "ci_low": 0.51,
      "ci_high": 0.65,
      "guard": {
        "share": 0.03,
        "threshold": 0.98,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.32,
      "stability": 0.08,
      "usd_per_1k_cases": 0.031,
      "median_latency_s": 1.24,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "nace-ai/drex-v1.5",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.55,
      "ci_low": 0.48,
      "ci_high": 0.61,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.43,
      "stability": 0,
      "usd_per_1k_cases": 0.0117,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    }
  ],
  "tests": "Exact paired McNemar tests between each pair of the top five models, with Holm correction across all 40 pairs in the four jobs. The leader is the model with the highest right, excluding rules and baselines. tie_group is 1 when the leader is not beaten, meaning p Holm against the leader is 0.05 or higher, and for the leader itself. A tested model with p Holm below 0.05 gets tie_group null and a note. A model outside the top five has no paired test in the source, so it gets tie_group null and a note saying whether its Wilson 95% interval overlaps the leader's. Rules and baselines get tie_group null. We never infer a tie from overlapping intervals.",
  "caveats": [
    "100 records are the CRM record's own profile (67 at the same company, 33 after a job change). 100 are namesakes proven different by LinkedIn member id (10 at the same company, 50 in the same industry, 40 elsewhere).",
    "Two free rules were built by looking at these records, so their scores run high. One is \"the CRM company, an email label or the exact title appears on the profile\" (90%). The other is \"the exact title alone\" (86%).",
    "On the 10 namesakes at the same company, none of the top five models got more than 1 right.",
    "Among the \"different\" records, at least 31 of 100 CRM companies came from two support and customer-success tools, against almost none among the \"same\" records. Models cannot see the source. Any traces of it are worth at most 3 points."
  ],
  "not_measured": [
    {
      "item": "Records from other companies",
      "reason": "The free rules read signals these records give away. Real CRM data may carry weaker signals or none."
    },
    {
      "item": "Other question wordings",
      "reason": "We froze the wording before any paid call and ran one version."
    }
  ]
}