{
  "benchmark": {
    "id": "hydra-balanced360-v2-20261023-authoring-r1",
    "manifest_sha256": "16bcaea16361f9ae0224ae90cfc8cb7e4287bcc4bc43ddd0cd2ffcc200431bdd",
    "role": "authoring_dev",
    "snapshot_generated_at": "2026-10-23T19:41:48.995714+00:00",
    "snapshot_local": "23.10.2026 · 21:41 MESZ",
    "timezone": "Europe/Berlin",
    "tasks": 360,
    "arms": 7,
    "started_cells": 2520,
    "competencies": 9,
    "tasks_per_competency": 40,
    "families": 270,
    "matched_coverage_eligible": true,
    "strict_metric": "Binary task-criteria and required-format fulfillment, failed transports included in the 360-start denominator",
    "ci95_method": "paired_stratified_family_bootstrap",
    "ci95_draws": 2000,
    "ci95_seed": 20261023,
    "ci95_scope": "Observed paired tasks only; authoring/development, no independent holdout",
    "runtime_scope": "longitudinal_campaign_multiple_runtime_epochs",
    "independent_holdout": false,
    "general_intelligence_index": false
  },
  "arms": [
    {
      "id": "hydra-performance",
      "name": "Hydra Performance",
      "kind": "hydra",
      "comparison_role": "main_complete_cohort",
      "effort": "production_performance_policy_no_override",
      "strict": {
        "passed": 344,
        "total": 360,
        "complete": 360,
        "transport_failures_in_denominator": 0,
        "percent": 95.55555555555556,
        "ci95_percent": [
          93.02953116118661,
          97.69660109202643
        ],
        "ci_method": "paired_stratified_family_bootstrap"
      },
      "format_repaired": {
        "passed": 351,
        "total": 360,
        "percent": 97.5,
        "ci95_percent": null,
        "evaluation": "posthoc_offline_format_replay_same_answers"
      },
      "costs": {
        "currency": "USD",
        "started_tasks": 360,
        "worker_known_usd": "0.039736380",
        "coordinator_known_usd": "0.211971360",
        "provider_known_usd": "0.251707740",
        "provider_total_usd": "0.251707740",
        "is_lower_bound": false,
        "unknown_cost_cases": 0,
        "eligible_for_complete_cost_ranking": true
      },
      "controlled_latency": {
        "observations": 9,
        "final_output_median_seconds": 2.560163457999999,
        "final_output_observed_range_seconds": [
          1.8423332499999996,
          3.8317877500000033
        ],
        "first_answer_text_median_seconds": 2.4783970000000006,
        "p95_seconds": null
      }
    },
    {
      "id": "hydra-performance-oss",
      "name": "Hydra OSS",
      "kind": "hydra",
      "comparison_role": "supplementary_incomplete_oss_diagnostic",
      "effort": "production_performance_policy_no_override",
      "strict": {
        "passed": 291,
        "total": 360,
        "complete": 347,
        "transport_failures_in_denominator": 13,
        "percent": 80.83333333333333,
        "ci95_percent": [
          76.80042232072077,
          84.7161673462219
        ],
        "ci_method": "paired_stratified_family_bootstrap"
      },
      "format_repaired": {
        "passed": 308,
        "total": 360,
        "percent": 85.55555555555556,
        "ci95_percent": null,
        "evaluation": "posthoc_offline_format_replay_same_answers"
      },
      "costs": {
        "currency": "USD",
        "started_tasks": 360,
        "worker_known_usd": "0.194642144",
        "coordinator_known_usd": "0.204838080",
        "provider_known_usd": "0.399480224",
        "provider_total_usd": null,
        "is_lower_bound": true,
        "unknown_cost_cases": 12,
        "eligible_for_complete_cost_ranking": false
      },
      "controlled_latency": {
        "observations": 9,
        "final_output_median_seconds": 2.0688383749999995,
        "final_output_observed_range_seconds": [
          1.790081083000001,
          3.0701030419999986
        ],
        "first_answer_text_median_seconds": 1.9904270829999986,
        "p95_seconds": null
      }
    },
    {
      "id": "gpt-6.1-sol-max-r2",
      "name": "GPT-6.1 Sol",
      "kind": "direct_model",
      "comparison_role": "main_complete_cohort",
      "effort": "maximum_supported_reasoning",
      "strict": {
        "passed": 355,
        "total": 360,
        "complete": 360,
        "transport_failures_in_denominator": 0,
        "percent": 98.61111111111111,
        "ci95_percent": [
          97.37816476194477,
          99.69156656656656
        ],
        "ci_method": "paired_stratified_family_bootstrap"
      },
      "format_repaired": {
        "passed": 359,
        "total": 360,
        "percent": 99.72222222222223,
        "ci95_percent": null,
        "evaluation": "posthoc_offline_format_replay_same_answers"
      },
      "costs": {
        "currency": "USD",
        "started_tasks": 360,
        "worker_known_usd": "1.457326200",
        "coordinator_known_usd": "0.000000000",
        "provider_known_usd": "1.457326200",
        "provider_total_usd": "1.457326200",
        "is_lower_bound": false,
        "unknown_cost_cases": 0,
        "eligible_for_complete_cost_ranking": true
      },
      "controlled_latency": {
        "observations": 9,
        "final_output_median_seconds": 3.086371374999999,
        "final_output_observed_range_seconds": [
          2.470431625000001,
          6.170967375
        ],
        "first_answer_text_median_seconds": 3.036805333,
        "p95_seconds": null
      }
    },
    {
      "id": "claude-sonnet-5.5-max-bedrock-repair-r3",
      "name": "Claude Sonnet 5.5",
      "kind": "direct_model",
      "comparison_role": "main_complete_cohort",
      "effort": "maximum_supported_reasoning",
      "strict": {
        "passed": 343,
        "total": 360,
        "complete": 360,
        "transport_failures_in_denominator": 0,
        "percent": 95.27777777777777,
        "ci95_percent": [
          92.90064536863316,
          97.4025577328423
        ],
        "ci_method": "paired_stratified_family_bootstrap"
      },
      "format_repaired": {
        "passed": 346,
        "total": 360,
        "percent": 96.11111111111111,
        "ci95_percent": null,
        "evaluation": "posthoc_offline_format_replay_same_answers"
      },
      "costs": {
        "currency": "USD",
        "started_tasks": 360,
        "worker_known_usd": "6.126228000",
        "coordinator_known_usd": "0.000000000",
        "provider_known_usd": "6.126228000",
        "provider_total_usd": "6.126228000",
        "is_lower_bound": false,
        "unknown_cost_cases": 0,
        "eligible_for_complete_cost_ranking": true
      },
      "controlled_latency": {
        "observations": 9,
        "final_output_median_seconds": 3.3560311660000006,
        "final_output_observed_range_seconds": [
          2.028392250000001,
          5.172679541999998
        ],
        "first_answer_text_median_seconds": 3.3051502079999993,
        "p95_seconds": null
      }
    },
    {
      "id": "mimo-v2.6-pro-max-xiaomi-r5",
      "name": "MiMo V2.6 Pro",
      "kind": "direct_model",
      "comparison_role": "main_complete_cohort",
      "effort": "maximum_supported_reasoning",
      "strict": {
        "passed": 260,
        "total": 360,
        "complete": 360,
        "transport_failures_in_denominator": 0,
        "percent": 72.22222222222223,
        "ci95_percent": [
          67.91894324300586,
          76.53550021744127
        ],
        "ci_method": "paired_stratified_family_bootstrap"
      },
      "format_repaired": {
        "passed": 328,
        "total": 360,
        "percent": 91.11111111111111,
        "ci95_percent": null,
        "evaluation": "posthoc_offline_format_replay_same_answers"
      },
      "costs": {
        "currency": "USD",
        "started_tasks": 360,
        "worker_known_usd": "0.124623150",
        "coordinator_known_usd": "0.000000000",
        "provider_known_usd": "0.124623150",
        "provider_total_usd": "0.124623150",
        "is_lower_bound": false,
        "unknown_cost_cases": 0,
        "eligible_for_complete_cost_ranking": true
      },
      "controlled_latency": {
        "observations": 9,
        "final_output_median_seconds": 4.198164667000001,
        "final_output_observed_range_seconds": [
          1.545074582999998,
          28.171504209000002
        ],
        "first_answer_text_median_seconds": 3.6742715840000018,
        "p95_seconds": null
      }
    },
    {
      "id": "deepseek-v4.1-flash-1017-max-r2",
      "name": "DeepSeek V4.1 Flash 1017",
      "kind": "direct_model",
      "comparison_role": "main_complete_cohort",
      "effort": "maximum_supported_reasoning",
      "strict": {
        "passed": 347,
        "total": 360,
        "complete": 360,
        "transport_failures_in_denominator": 0,
        "percent": 96.38888888888889,
        "ci95_percent": [
          93.89732783725032,
          98.392501017501
        ],
        "ci_method": "paired_stratified_family_bootstrap"
      },
      "format_repaired": {
        "passed": 354,
        "total": 360,
        "percent": 98.33333333333333,
        "ci95_percent": null,
        "evaluation": "posthoc_offline_format_replay_same_answers"
      },
      "costs": {
        "currency": "USD",
        "started_tasks": 360,
        "worker_known_usd": "0.095008100",
        "coordinator_known_usd": "0.000000000",
        "provider_known_usd": "0.095008100",
        "provider_total_usd": "0.095008100",
        "is_lower_bound": false,
        "unknown_cost_cases": 0,
        "eligible_for_complete_cost_ranking": true
      },
      "controlled_latency": {
        "observations": 9,
        "final_output_median_seconds": 1.7502088330000005,
        "final_output_observed_range_seconds": [
          0.9456090000000004,
          2.395496625
        ],
        "first_answer_text_median_seconds": 1.7471062500000007,
        "p95_seconds": null
      }
    },
    {
      "id": "gpt-6-luna-max-r2",
      "name": "GPT-6 Luna",
      "kind": "direct_model",
      "comparison_role": "main_complete_cohort",
      "effort": "maximum_supported_reasoning",
      "strict": {
        "passed": 347,
        "total": 360,
        "complete": 360,
        "transport_failures_in_denominator": 0,
        "percent": 96.38888888888889,
        "ci95_percent": [
          94.05423135686294,
          98.39320171094845
        ],
        "ci_method": "paired_stratified_family_bootstrap"
      },
      "format_repaired": {
        "passed": 353,
        "total": 360,
        "percent": 98.05555555555556,
        "ci95_percent": null,
        "evaluation": "posthoc_offline_format_replay_same_answers"
      },
      "costs": {
        "currency": "USD",
        "started_tasks": 360,
        "worker_known_usd": "0.048016210",
        "coordinator_known_usd": "0.000000000",
        "provider_known_usd": "0.048016210",
        "provider_total_usd": "0.048016210",
        "is_lower_bound": false,
        "unknown_cost_cases": 0,
        "eligible_for_complete_cost_ranking": true
      },
      "controlled_latency": {
        "observations": 9,
        "final_output_median_seconds": 2.0854975419999975,
        "final_output_observed_range_seconds": [
          1.4787651250000018,
          2.9346951660000014
        ],
        "first_answer_text_median_seconds": 2.072109333,
        "p95_seconds": null
      }
    }
  ],
  "comparison_roles": {
    "main_complete_cohort": {
      "arm_ids": [
        "hydra-performance",
        "gpt-6.1-sol-max-r2",
        "claude-sonnet-5.5-max-bedrock-repair-r3",
        "mimo-v2.6-pro-max-xiaomi-r5",
        "deepseek-v4.1-flash-1017-max-r2",
        "gpt-6-luna-max-r2"
      ],
      "arms": 6,
      "tasks_per_arm": 360,
      "started_cells": 2160,
      "source_section": "report-r2/results.json:primary_comparison",
      "complete_response_and_cost_coverage": true
    },
    "supplementary_incomplete_oss_diagnostic": {
      "arm_id": "hydra-performance-oss",
      "started": 360,
      "complete": 347,
      "strict_passed": 291,
      "transport_failures_in_denominator": 13,
      "unknown_cost_cases": 12,
      "not_equal_complete_qualification": true,
      "exclusion_basis": "The original report builder explicitly excludes OSS from primary_comparison; its README defines six complete arms and its comparison explorer calls OSS provisional. OSS has incomplete response and cost coverage, unlike the six complete primary arms.",
      "permitted_descriptive_scope": "The later final-review candidate explicitly lists all seven arms on identical 360-task starts and retains 13 OSS failures in the denominator. A clearly labeled supplementary 291/360 strict-fulfillment result is supported; it is not a complete transport/cost qualification, full price, savings claim or survival-selected 347-task comparison.",
      "website_label_de": "Hydra OSS · ergänzender Diagnosearm · 347/360 Antworten, 13 Transportfehler, 12 Kostenfälle offen",
      "website_label_en": "Hydra OSS · supplementary diagnostic arm · 347/360 responses, 13 transport failures, 12 cost cases unresolved"
    }
  },
  "model_mix": {
    "arm_id": "hydra-performance",
    "scope": "Historical Balanced360 main-run selections, not present-day customer usage",
    "total": 360,
    "complete": 360,
    "items": [
      {
        "name": "DeepSeek V4.1 Flash 1017",
        "model": "deepseek/deepseek-v4.1-flash-20261017",
        "count": 200,
        "percent": 55.55555555555556
      },
      {
        "name": "GPT-6 Luna",
        "model": "openai/gpt-6-luna-20260922",
        "count": 160,
        "percent": 44.44444444444444
      }
    ]
  },
  "cost_split": {
    "arm_id": "hydra-performance",
    "started_tasks": 360,
    "currency": "USD",
    "worker_usd": "0.039736380",
    "coordinator_usd": "0.211971360",
    "provider_total_usd": "0.251707740",
    "worker_percent_of_provider_total": 15.786713590928908,
    "coordinator_percent_of_provider_total": 84.21328640907109,
    "is_time_series": false,
    "scope": "Measured provider-model costs including routing coordinator; excludes external services and infrastructure; not a customer price"
  },
  "controlled_latency": {
    "snapshot_local": "24.10.2026 · 00:01 MESZ",
    "timezone": "Europe/Berlin",
    "tasks": 3,
    "repetitions_per_task": 3,
    "observations_per_arm": 9,
    "unit": "seconds",
    "metric": "Median from actual API dispatch to final output completion",
    "first_answer_text_definition": "First answer text, not initial reasoning tokens",
    "excludes": [
      "client queue time",
      "typing",
      "browser rendering"
    ],
    "scope": "Separate controlled descriptive sample, not main-run latency or a general ranking; format failures remain included; runtime evidence spans a planned refresh",
    "p95_available": false
  },
  "format_replay": {
    "created_at": "2026-10-26T10:21:07.401274+00:00",
    "scope": "Post-hoc offline artifact replay, all seven arms on unchanged 360 tasks. Not a live product fix or new model benchmark. Original strict scores preserved.",
    "new_model_calls": 0,
    "cost_latency_transport_status_unchanged": true,
    "ci95_available": false,
    "strict_confidence_intervals_must_not_be_reused": true
  },
  "limitations": [
    "Internal authoring/development tasks, no independent holdout and no general intelligence score.",
    "Six complete arms form primary_comparison; the incomplete OSS arm is a separately labeled supplementary diagnostic. Shared task coverage does not imply equal complete qualification.",
    "The seven-arm campaign spans multiple runtime epochs; it is not a synchronized present-day live-service benchmark.",
    "Direct comparators use maximum supported reasoning; Hydra uses its performance policy without an effort override.",
    "Hydra OSS retains 13 transport failures in its 360-start denominator; twelve cost cases are unknown, so its known sums are lower bounds.",
    "Provider costs include coordinator where applicable, but exclude external services and infrastructure; they are not current tariffs or complete business cost.",
    "Controlled latency is a separate nine-observation-per-arm descriptive sample; no defensible p95 or general speed promise is claimed.",
    "Format repair is a post-hoc offline replay of the same answers, not a new run, live product repair or source of strict confidence intervals.",
    "Model-selection illustration and public dashboard simulation remain labeled schema/demo; no synthetic week trend or customer-usage values are inferred from these aggregates.",
    "The blocked Hydra=100/LiveBench calibration is not used."
  ],
  "schema": "hydra-website-benchmark-v1",
  "presentation_cost_basis": {
    "component": "worker_known_usd",
    "coordinator_included": false,
    "scope": "Recorded response-model cost; provider totals are retained separately for provenance"
  },
  "prepared_on": "2026-11-02",
  "source_sha256": {
    "report-r2/results.json": "81c659f578f8a502cb8735a913112b09c04b9982e7e1efcf1107a0547b9b0106",
    "format-repair-20261026-v1/results.json": "1ed7bbc6c9682f0df580ffc76ac56e399555e6731c56ae798433b31c02a43e49",
    "format-repair-20261026-v1/site/performance.fragment.html": "0c41620fb649a6c6c3f0be83acfc16b726ffedaa46614df2c8f0b5f4029ea307",
    "format-repair-20261026-v1/publication.json": "dc4c9b92b929f43de02acf348f3a49b8478b598f6fccb049cf7642778d291b3d",
    "report-r2/README.md": "9cc8e13f25cd46b830c6e9ba53c8c7ef703ed792d2f17678955e5a5eef14969c",
    "report/build_report.py": "195171916f232d820220e7fbbcc4b751f188a8088ea87af5266d9862c123fa1a",
    "report-r2/data-handoff-20261026-v1/oss-gap-audit-20261026.md": "07d983c7afc2388933129ee0c2eaf96ae6106034039b069f914b0fef7d8ca7f4",
    "final-review-current-v1/ABSCHLUSS-KANDIDAT.md": "85d116013a16662de8bdb9b3b6fab9f4a4f6e1a1e469757b6d311d1b74365ac2"
  },
  "historical_publication": "https://hydra-performance-review-oct23.nils524686.chatgpt.site"
}