{
  "name": "HorizonSWE",
  "release": "Pilot 01",
  "released_on": "2026-10-14",
  "status": "Preliminary research pilot",
  "task_count": 5,
  "repository_count": 3,
  "language": "Python",
  "models": [
    {
      "id": "anthropic/claude-sonnet-5@default",
      "name": "Claude Sonnet 5",
      "provider": "Anthropic",
      "monogram": "A",
      "study": "core"
    },
    {
      "id": "google/gemini-3-flash-preview",
      "name": "Gemini 3 Flash",
      "provider": "Google",
      "monogram": "G",
      "study": "core"
    },
    {
      "id": "openai/gpt-oss-120b",
      "name": "GPT-oss 120B",
      "provider": "OpenAI",
      "monogram": "O",
      "study": "core"
    },
    {
      "id": "qwen/qwen3-next-80b-a3b-instruct",
      "name": "Qwen3 Next 80B",
      "provider": "Qwen",
      "monogram": "Q",
      "study": "core"
    },
    {
      "id": "deepseek-ai/deepseek-r1-0528",
      "name": "DeepSeek R1",
      "provider": "DeepSeek",
      "monogram": "D",
      "study": "core"
    },
    {
      "id": "openai/gpt-5.4-nano-2026-03-17",
      "name": "GPT-5.4 Nano",
      "provider": "OpenAI",
      "monogram": "O",
      "study": "core"
    },
    {
      "id": "google/gemini-3.1-flash-lite-preview",
      "name": "Gemini 3.1 Flash Lite",
      "provider": "Google",
      "monogram": "G",
      "study": "core"
    },
    {
      "id": "ibm/granite-4.0-h-small",
      "name": "Granite 4.0 Small",
      "provider": "IBM",
      "monogram": "I",
      "study": "core"
    },
    {
      "id": "google/gemini-3.5-flash-lite",
      "name": "Gemini 3.5 Flash Lite",
      "provider": "Google",
      "monogram": "G",
      "study": "compact"
    },
    {
      "id": "google/gemini-3.6-flash",
      "name": "Gemini 3.6 Flash",
      "provider": "Google",
      "monogram": "G",
      "study": "compact"
    }
  ],
  "studies": {
    "core": {
      "label": "Core pilot",
      "model_count": 8,
      "task_count": 5,
      "attempt_count": 40,
      "max_turns": 40,
      "max_output_tokens_per_request": 16384,
      "protocol": "retrieval-pilot-v4",
      "agent_sha256": "4311b3830f28e6f81a1fd1fff589a9fe7a943e203529d4c5619f6e7af1ceab1b",
      "adapter_sha256": "8acfe5cec3d53a63fe6ce9266a4a9df540a4aad2b0655b1fd45699a5aecec2d8"
    },
    "compact": {
      "label": "Compact extension",
      "model_count": 2,
      "task_count": 1,
      "attempt_count": 2,
      "max_turns": 20,
      "max_output_tokens_per_request": 2048,
      "protocol": "compact-catalog-v1",
      "agent_sha256": "ecbb40202b99ea6fbaa3f516e8c7c2b1d97c8d6dbeb64fa68c4577a49f82ccce",
      "adapter_sha256": "383aab331f329150fef55e45b630d0f500754d20d53b56a2d6fcae0527e24e94"
    }
  },
  "tasks": [
    {
      "id": "FL-006",
      "theme": "Tool registry lifecycle"
    },
    {
      "id": "FL-002",
      "theme": "Bundle installation"
    },
    {
      "id": "FL-004",
      "theme": "Configuration overlays"
    },
    {
      "id": "FL-005",
      "theme": "Event inspection"
    },
    {
      "id": "FL-007",
      "theme": "Per-step runtime configuration"
    }
  ],
  "controls": [
    {
      "task": "FL-006",
      "reference_patch": "passed",
      "comment_only_patch": "failed"
    },
    {
      "task": "FL-002",
      "reference_patch": "passed",
      "comment_only_patch": "failed"
    },
    {
      "task": "FL-004",
      "reference_patch": "passed",
      "comment_only_patch": "failed"
    },
    {
      "task": "FL-005",
      "reference_patch": "passed",
      "comment_only_patch": "failed"
    },
    {
      "task": "FL-007",
      "reference_patch": "passed",
      "comment_only_patch": "failed"
    }
  ],
  "account_credit_snapshot": {
    "reported_used_usd": 8.87667531,
    "limit_usd": 10,
    "includes": "Earlier diagnostics, catalog access probes and recorded inference attempts."
  },
  "catalog_access": {
    "additional_aliases_checked": 36,
    "accessible": 2,
    "unavailable": 34
  },
  "limitations": [
    "Small, selected Python-focused five-task pilot; not a representative sample of software engineering.",
    "One attempt per cell; no reliable model ranking or general capability inference.",
    "The same pilot tasks informed harness debugging; no held-out harness-development split was used.",
    "Model aliases may change; reasoning and tool interfaces differ by backend, with provider-default sampling.",
    "Kaggle auth issued a staging-host proxy endpoint; provider identity/version needs confirmation before publication.",
    "Access, infrastructure and budget failures are distinct from task failures.",
    "All original tasks still require independent publication-readiness review.",
    "Hidden test coverage and reference validation do not establish exhaustive correctness.",
    "All earlier smoke and v1-v3 diagnostic runs are excluded from this protocol."
  ],
  "records": [
    {
      "model": "anthropic/claude-sonnet-5@default",
      "task": "FL-006",
      "status": "failed",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 646.5,
      "reported_inference_credit_usd": 1.489365,
      "base_tests_passed": 3,
      "feature_tests_passed": 18,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 65228,
      "input_tokens": 510681,
      "responses_with_usage": 40,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "anthropic/claude-sonnet-5@default",
      "task": "FL-002",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 132.41,
      "reported_inference_credit_usd": 1.062874,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 7932,
      "input_tokens": 689293,
      "responses_with_usage": 40,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "anthropic/claude-sonnet-5@default",
      "task": "FL-004",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 658.48,
      "reported_inference_credit_usd": 1.308346,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 53237,
      "input_tokens": 484342,
      "responses_with_usage": 40,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "anthropic/claude-sonnet-5@default",
      "task": "FL-005",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 119.65,
      "reported_inference_credit_usd": 0.811273,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 5805,
      "input_tokens": 482796,
      "responses_with_usage": 40,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "anthropic/claude-sonnet-5@default",
      "task": "FL-007",
      "status": "budget_limited",
      "generation_status": "budget_limit",
      "turns": 13,
      "seconds": 49.4,
      "reported_inference_credit_usd": 0.184253,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 2605,
      "input_tokens": 130192,
      "responses_with_usage": 13,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3-flash-preview",
      "task": "FL-006",
      "status": "failed",
      "generation_status": "completed",
      "turns": 33,
      "seconds": 99.78,
      "reported_inference_credit_usd": 0.145463,
      "base_tests_passed": 4,
      "feature_tests_passed": 15,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 9088,
      "input_tokens": 245389,
      "responses_with_usage": 33,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3-flash-preview",
      "task": "FL-002",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 85.96,
      "reported_inference_credit_usd": 0.185446,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 1228,
      "input_tokens": 377951,
      "responses_with_usage": 40,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3-flash-preview",
      "task": "FL-004",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 233.34,
      "reported_inference_credit_usd": 0.299593,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 39034,
      "input_tokens": 426948,
      "responses_with_usage": 40,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3-flash-preview",
      "task": "FL-005",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 80.32,
      "reported_inference_credit_usd": 0.092334,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 989,
      "input_tokens": 187659,
      "responses_with_usage": 40,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3-flash-preview",
      "task": "FL-007",
      "status": "budget_limited",
      "generation_status": "budget_limit",
      "turns": 22,
      "seconds": 46.43,
      "reported_inference_credit_usd": 0.074878,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 1025,
      "input_tokens": 152614,
      "responses_with_usage": 22,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "openai/gpt-oss-120b",
      "task": "FL-006",
      "status": "empty_response",
      "generation_status": "empty_proxy_message",
      "turns": 1,
      "seconds": 129.96,
      "reported_inference_credit_usd": 7.5e-05,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 38,
      "input_tokens": 677,
      "responses_with_usage": 1,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "openai/gpt-oss-120b",
      "task": "FL-002",
      "status": "empty_response",
      "generation_status": "empty_proxy_message",
      "turns": 1,
      "seconds": 2.48,
      "reported_inference_credit_usd": 9.2e-05,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 37,
      "input_tokens": 874,
      "responses_with_usage": 1,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "openai/gpt-oss-120b",
      "task": "FL-004",
      "status": "empty_response",
      "generation_status": "empty_proxy_message",
      "turns": 1,
      "seconds": 3.94,
      "reported_inference_credit_usd": 7.3e-05,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 39,
      "input_tokens": 658,
      "responses_with_usage": 1,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "openai/gpt-oss-120b",
      "task": "FL-005",
      "status": "empty_response",
      "generation_status": "empty_proxy_message",
      "turns": 1,
      "seconds": 125.76,
      "reported_inference_credit_usd": 8.1e-05,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 32,
      "input_tokens": 771,
      "responses_with_usage": 1,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "openai/gpt-oss-120b",
      "task": "FL-007",
      "status": "empty_response",
      "generation_status": "empty_proxy_message",
      "turns": 1,
      "seconds": 139.82,
      "reported_inference_credit_usd": 0.0001,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 39,
      "input_tokens": 952,
      "responses_with_usage": 1,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "qwen/qwen3-next-80b-a3b-instruct",
      "task": "FL-006",
      "status": "infrastructure_error",
      "generation_status": "infrastructure_error",
      "turns": 3,
      "seconds": 173.09,
      "reported_inference_credit_usd": 0.000668,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": "RateLimitError",
      "http_status": 429,
      "study": "core",
      "output_tokens": 130,
      "input_tokens": 3412,
      "responses_with_usage": 3,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "qwen/qwen3-next-80b-a3b-instruct",
      "task": "FL-002",
      "status": "infrastructure_error",
      "generation_status": "infrastructure_error",
      "turns": 33,
      "seconds": 961.8,
      "reported_inference_credit_usd": 0.034093,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": "RateLimitError",
      "http_status": 429,
      "study": "core",
      "output_tokens": 1634,
      "input_tokens": 214215,
      "responses_with_usage": 33,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "qwen/qwen3-next-80b-a3b-instruct",
      "task": "FL-004",
      "status": "infrastructure_error",
      "generation_status": "infrastructure_error",
      "turns": 11,
      "seconds": 523.11,
      "reported_inference_credit_usd": 0.006518,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": "RateLimitError",
      "http_status": 429,
      "study": "core",
      "output_tokens": 1391,
      "input_tokens": 32326,
      "responses_with_usage": 11,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "qwen/qwen3-next-80b-a3b-instruct",
      "task": "FL-005",
      "status": "no_patch",
      "generation_status": "time_limit",
      "turns": 38,
      "seconds": 1205.31,
      "reported_inference_credit_usd": 0.056887,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 1890,
      "input_tokens": 364126,
      "responses_with_usage": 38,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "qwen/qwen3-next-80b-a3b-instruct",
      "task": "FL-007",
      "status": "budget_limited",
      "generation_status": "budget_limit",
      "turns": 18,
      "seconds": 477.14,
      "reported_inference_credit_usd": 0.01392,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 842,
      "input_tokens": 86062,
      "responses_with_usage": 18,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "deepseek-ai/deepseek-r1-0528",
      "task": "FL-006",
      "status": "format_error",
      "generation_status": "format_error",
      "turns": 1,
      "seconds": 110.13,
      "reported_inference_credit_usd": 0.006368,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 936,
      "input_tokens": 973,
      "responses_with_usage": 1,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "deepseek-ai/deepseek-r1-0528",
      "task": "FL-002",
      "status": "format_error",
      "generation_status": "format_error",
      "turns": 1,
      "seconds": 4.84,
      "reported_inference_credit_usd": 0.002874,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 236,
      "input_tokens": 1185,
      "responses_with_usage": 1,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "deepseek-ai/deepseek-r1-0528",
      "task": "FL-004",
      "status": "no_patch",
      "generation_status": "completed",
      "turns": 14,
      "seconds": 967.6,
      "reported_inference_credit_usd": 0.544347,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 93633,
      "input_tokens": 28688,
      "responses_with_usage": 14,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "deepseek-ai/deepseek-r1-0528",
      "task": "FL-005",
      "status": "failed",
      "generation_status": "format_error",
      "turns": 15,
      "seconds": 604.01,
      "reported_inference_credit_usd": 0.213257,
      "base_tests_passed": 0,
      "feature_tests_passed": 0,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 26399,
      "input_tokens": 52372,
      "responses_with_usage": 15,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "deepseek-ai/deepseek-r1-0528",
      "task": "FL-007",
      "status": "budget_limited",
      "generation_status": "budget_limit",
      "turns": 7,
      "seconds": 169.9,
      "reported_inference_credit_usd": 0.083241,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 12335,
      "input_tokens": 12320,
      "responses_with_usage": 7,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "openai/gpt-5.4-nano-2026-03-17",
      "task": "FL-006",
      "status": "failed",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 97.52,
      "reported_inference_credit_usd": 0.049286,
      "base_tests_passed": 3,
      "feature_tests_passed": 6,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 5288,
      "input_tokens": 277202,
      "responses_with_usage": 40,
      "interface": "native tools; reasoning none"
    },
    {
      "model": "openai/gpt-5.4-nano-2026-03-17",
      "task": "FL-002",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 90.96,
      "reported_inference_credit_usd": 0.056328,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 1982,
      "input_tokens": 341945,
      "responses_with_usage": 40,
      "interface": "native tools; reasoning none"
    },
    {
      "model": "openai/gpt-5.4-nano-2026-03-17",
      "task": "FL-004",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 205.36,
      "reported_inference_credit_usd": 0.066638,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 1370,
      "input_tokens": 389486,
      "responses_with_usage": 40,
      "interface": "native tools; reasoning none"
    },
    {
      "model": "openai/gpt-5.4-nano-2026-03-17",
      "task": "FL-005",
      "status": "no_patch",
      "generation_status": "turn_limit",
      "turns": 40,
      "seconds": 90.66,
      "reported_inference_credit_usd": 0.049164,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 1364,
      "input_tokens": 334291,
      "responses_with_usage": 40,
      "interface": "native tools; reasoning none"
    },
    {
      "model": "openai/gpt-5.4-nano-2026-03-17",
      "task": "FL-007",
      "status": "budget_limited",
      "generation_status": "budget_limit",
      "turns": 3,
      "seconds": 10.45,
      "reported_inference_credit_usd": 0.002309,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 400,
      "input_tokens": 10542,
      "responses_with_usage": 3,
      "interface": "native tools; reasoning none"
    },
    {
      "model": "google/gemini-3.1-flash-lite-preview",
      "task": "FL-006",
      "status": "failed",
      "generation_status": "completed",
      "turns": 19,
      "seconds": 59.53,
      "reported_inference_credit_usd": 0.045609,
      "base_tests_passed": 0,
      "feature_tests_passed": 0,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 10811,
      "input_tokens": 141052,
      "responses_with_usage": 19,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3.1-flash-lite-preview",
      "task": "FL-002",
      "status": "failed",
      "generation_status": "completed",
      "turns": 35,
      "seconds": 93.82,
      "reported_inference_credit_usd": 0.081244,
      "base_tests_passed": 115,
      "feature_tests_passed": 0,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 9579,
      "input_tokens": 276449,
      "responses_with_usage": 35,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3.1-flash-lite-preview",
      "task": "FL-004",
      "status": "failed",
      "generation_status": "completed",
      "turns": 16,
      "seconds": 65.58,
      "reported_inference_credit_usd": 0.047971,
      "base_tests_passed": 172,
      "feature_tests_passed": 0,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 13847,
      "input_tokens": 139659,
      "responses_with_usage": 16,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3.1-flash-lite-preview",
      "task": "FL-005",
      "status": "failed",
      "generation_status": "completed",
      "turns": 32,
      "seconds": 74.8,
      "reported_inference_credit_usd": 0.078706,
      "base_tests_passed": 3,
      "feature_tests_passed": 15,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 8433,
      "input_tokens": 285862,
      "responses_with_usage": 32,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "google/gemini-3.1-flash-lite-preview",
      "task": "FL-007",
      "status": "budget_limited",
      "generation_status": "budget_limit",
      "turns": 13,
      "seconds": 30.15,
      "reported_inference_credit_usd": 0.011193,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 1376,
      "input_tokens": 58186,
      "responses_with_usage": 13,
      "interface": "native tools; low reasoning"
    },
    {
      "model": "ibm/granite-4.0-h-small",
      "task": "FL-006",
      "status": "format_error",
      "generation_status": "format_error",
      "turns": 2,
      "seconds": 6.9,
      "reported_inference_credit_usd": 0.003061,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 984,
      "input_tokens": 2002,
      "responses_with_usage": 2,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "ibm/granite-4.0-h-small",
      "task": "FL-002",
      "status": "no_patch",
      "generation_status": "completed",
      "turns": 1,
      "seconds": 5.51,
      "reported_inference_credit_usd": 0.001979,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 652,
      "input_tokens": 1164,
      "responses_with_usage": 1,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "ibm/granite-4.0-h-small",
      "task": "FL-004",
      "status": "no_patch",
      "generation_status": "completed",
      "turns": 4,
      "seconds": 23.6,
      "reported_inference_credit_usd": 0.011204,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 3984,
      "input_tokens": 4148,
      "responses_with_usage": 4,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "ibm/granite-4.0-h-small",
      "task": "FL-005",
      "status": "no_patch",
      "generation_status": "completed",
      "turns": 4,
      "seconds": 18.63,
      "reported_inference_credit_usd": 0.007693,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 2521,
      "input_tokens": 4634,
      "responses_with_usage": 4,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "ibm/granite-4.0-h-small",
      "task": "FL-007",
      "status": "format_error",
      "generation_status": "format_error",
      "turns": 5,
      "seconds": 21.45,
      "reported_inference_credit_usd": 0.009512,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "core",
      "output_tokens": 3006,
      "input_tokens": 6658,
      "responses_with_usage": 5,
      "interface": "JSON actions; default reasoning"
    },
    {
      "model": "google/gemini-3.5-flash-lite",
      "task": "FL-006",
      "status": "no_patch",
      "generation_status": "output_limit",
      "turns": 11,
      "seconds": 37.29,
      "reported_inference_credit_usd": 0.019018,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "compact",
      "output_tokens": 3653,
      "input_tokens": 73027,
      "responses_with_usage": 11,
      "interface": "Native tools; low reasoning"
    },
    {
      "model": "google/gemini-3.6-flash",
      "task": "FL-006",
      "status": "no_patch",
      "generation_status": "output_limit",
      "turns": 13,
      "seconds": 72.33,
      "reported_inference_credit_usd": 0.07871,
      "base_tests_passed": null,
      "feature_tests_passed": null,
      "error_type": null,
      "http_status": null,
      "study": "compact",
      "output_tokens": 9738,
      "input_tokens": 154197,
      "responses_with_usage": 13,
      "interface": "Native tools; low reasoning"
    }
  ],
  "research_series": "Research 01"
}