{
  "publisher": "Kyde Research",
  "page": "https://kyde.com/research/",
  "measured": "2026-10-27",
  "method_version": "990bfef",
  "code": "https://github.com/kydehq/behavioral-drift-detection",
  "note": "Agent behavior on public corpora. Draft studies are not confirmed for publication.",
  "studies": [
    {
      "id": "model-switch",
      "status": "draft",
      "name": "Model changes and behavioral drift",
      "question": "Does changing the underlying model change how an agent behaves?",
      "finding": "A new model can change how the same agent works. Or barely at all.",
      "setup": "Tool-use distribution of the same agent framework on SWE-bench, before and after the model behind it changed. 0 means identical, 1 means nothing in common.",
      "columns": [
        "Framework",
        "Before",
        "After",
        "Runs after",
        "Tool-use shift (JSD)"
      ],
      "rows": [
        [
          "SWE-agent",
          "Claude 3 Opus, 2024-04",
          "GPT-4, 2024-04",
          496,
          "0.085"
        ],
        [
          "SWE-agent",
          "GPT-4, 2024-04",
          "Claude 3.5 Sonnet, 2024-06",
          500,
          "0.116"
        ],
        [
          "SWE-agent",
          "Claude 3.5 Sonnet, 2024-06",
          "GPT-4o, 2024-07",
          465,
          "0.091"
        ],
        [
          "SWE-agent",
          "GPT-4o, 2024-07",
          "SWE-agent-LM-32B, 2025-05",
          500,
          "0.813"
        ],
        [
          "OpenHands",
          "Claude 4 Sonnet, 2025-05",
          "Kimi K2, 2025-07",
          500,
          "0.002"
        ],
        [
          "OpenHands",
          "Kimi K2, 2025-07",
          "GPT-5, 2025-08",
          500,
          "0.105"
        ],
        [
          "Emergent",
          "2024-10",
          "2024-12, same setup",
          500,
          "0.000"
        ]
      ],
      "limitations": "Separate SWE-bench submissions, not one deployment over time. The last row is the control: no change, no shift.",
      "runs": 3461,
      "corpus": "SWE-bench submission logs",
      "source": "https://github.com/kydehq/behavioral-drift-detection/blob/main/experiments/results/swebench-two-regime.md"
    },
    {
      "id": "run-length",
      "status": "draft",
      "name": "Errors over the course of a run",
      "question": "Does the probability of failure increase with run length?",
      "finding": "Long runs fail more: errors rise twelvefold over the first 40 steps.",
      "setup": "Share of calls with an error status, by how far into the run they happen. One GAIA agent, 117 runs.",
      "columns": [
        "Steps into the run",
        "Calls",
        "Error status"
      ],
      "rows": [
        [
          "0 to 9",
          1170,
          "2.1 %"
        ],
        [
          "10 to 19",
          663,
          "7.1 %"
        ],
        [
          "20 to 29",
          409,
          "16.4 %"
        ],
        [
          "30 to 39",
          321,
          "25.5 %"
        ],
        [
          "40 to 49",
          237,
          "21.9 %"
        ],
        [
          "50 to 59",
          184,
          "15.2 %"
        ],
        [
          "60 to 69",
          143,
          "11.2 %"
        ]
      ],
      "limitations": "Later steps are left out: below 140 calls per bin the rate is noise.",
      "runs": 117,
      "corpus": "TRAIL, GAIA, o3-mini",
      "source": "https://github.com/kydehq/behavioral-drift-detection/blob/main/experiments/results/trail-context-decay.md"
    },
    {
      "id": "reward-hacking",
      "status": "draft",
      "name": "Reward hacking",
      "question": "When an agent games its grader, what gives it away?",
      "finding": "Agents that game the test say so in their reasoning.",
      "setup": "Terminal Wrench runs where the agent exploited the grader instead of solving the task. Detection from the agent's own reasoning text.",
      "columns": [
        "Agent model",
        "Hacked runs",
        "Caught from reasoning",
        "False alarms"
      ],
      "rows": [
        [
          "Claude Opus 4.6",
          590,
          "98.6 %",
          "0.2 %"
        ],
        [
          "Gemini 3.1 Pro",
          1181,
          "97.5 %",
          "1.2 %"
        ],
        [
          "GPT-5.4",
          783,
          "99.7 %",
          "0.5 %"
        ]
      ],
      "limitations": "These agents narrate their exploits openly. A ceiling, not a guarantee.",
      "runs": 4415,
      "corpus": "Terminal Wrench",
      "source": "https://github.com/kydehq/behavioral-drift-detection/blob/main/experiments/results/l3-terminalwrench-rewardhack.md"
    },
    {
      "id": "cli-updates",
      "status": "draft",
      "name": "Behavioral stability across tool updates",
      "question": "Do tool updates measurably change agent behavior?",
      "finding": "Across Claude Code updates, tool use shifted no more than where nothing was updated.",
      "setup": "Real Claude Code sessions of 171 users. Tool-use shift across 19 version updates, against the same measurement at points with no update.",
      "columns": [
        "Comparison",
        "Cases",
        "Median shift (JSD)"
      ],
      "rows": [
        [
          "Across a version update",
          19,
          "0.132"
        ],
        [
          "Control, no update",
          65,
          "0.127"
        ],
        [
          "Same user, adjacent periods",
          58,
          "0.149"
        ],
        [
          "Same user, first vs. last period",
          34,
          "0.211"
        ],
        [
          "Different users",
          1653,
          "0.358"
        ]
      ],
      "limitations": "A result we did not expect. Behavior within a user varies as much as across an update, while each user's agent keeps its own fingerprint: different users are more than twice as far apart. Baselines therefore have to be per agent, and they age within weeks.",
      "runs": 4771,
      "corpus": "SWE-chat, Claude Code sessions of 171 users",
      "source": "https://github.com/kydehq/behavioral-drift-detection/blob/main/experiments/results/swechat-real-timelines.md"
    }
  ]
}