"""Variant-selection contracts: fit the catalog's single Q4-class build to a machine and price it honestly. Pure decision-table tests over synthetic budgets.""" from __future__ import annotations from hermes_cli.local_runtime.catalog import ( CATALOG, catalog_by_id, find_entry_for_model, select_variant, ) from hermes_cli.local_runtime.estimator import HardwareBudget GIB = 1 << 30 def budget(vram_gib: float, ram_gib: float = 64) -> HardwareBudget: return HardwareBudget(usable_vram_bytes=int(vram_gib * GIB), total_device_bytes=int(vram_gib * GIB), ram_available_bytes=int(ram_gib * GIB)) def test_every_entry_ships_exactly_one_q4_build(): """No quant ladder: one Q4-class build per entry (K_M where the repo ships it, XL elsewhere) — the quant class current engines optimize for. Nothing below Q4 ever ships. Validation status is explicit per variant in catalog.json; unvalidated builds are permitted (day-0 entries) and surface as unbadged rows in the pane.""" for entry in CATALOG: assert len(entry.variants) == 1, ( f"{entry.id}: {len(entry.variants)} variants — expected exactly one") build = entry.variants[0] assert build.quant.startswith(("UD-Q4", "Q4")), ( f"{entry.id}: ships {build.quant}, not a Q4-class build") for asset in entry.download_files(build): assert asset.size_bytes > 0, f"{entry.id}: no size on {asset.path}" def test_split_variants_have_coherent_parts(): """Multi-file variants: same model_id from every part, exact sizes, first file is the load target.""" entry = catalog_by_id()["deepseek-v4-flash"] for v in entry.variants: assert len(v.files) >= 2, "deepseek ships split GGUFs" assert "00001-of" in v.files[0].path, "first part must be the load target" assert v.size_bytes == sum(f.size_bytes for f in v.files) assert entry.draft is not None, "DSpark draft rides along" def test_selection_is_the_q4_build_even_with_headroom(): """The selector picks the Q4 build even when bigger quants would fit with room to spare — headroom buys window, not quant. Larger builds stay one tile click away in the pane.""" entry = catalog_by_id()["qwen3.8-27b"] choice = select_variant(entry, budget(60)) assert choice is not None assert choice.zero_spill assert choice.variant.quant == entry.variants[-1].quant # the Q4 rung assert choice.reason_key == "best-large-window" def test_selected_build_constant_and_fit_shape_monotone_in_vram(): """More VRAM never changes the selected build (always the Q4 rung); what improves is the fit shape: spilled -> floor -> target window.""" entry = catalog_by_id()["qwen3.8-27b"] quants = set() shapes = [] rank = {"smallest-fits-spilled": 0, "best-fits": 1, "best-large-window": 2} for vram in (8, 12, 16, 24, 32, 48): choice = select_variant(entry, budget(vram)) assert choice is not None quants.add(choice.variant.quant) shapes.append(rank[choice.reason_key]) assert quants == {entry.variants[-1].quant}, f"selection not constant: {quants}" assert shapes == sorted(shapes), f"fit shape not monotone in VRAM: {shapes}" def test_small_card_gets_q4_spilled_never_below(): """8 GiB card + 27B: nothing zero-spills. The floor holds — the selector offers Q4 spilled (priced honestly), never a sub-Q4 build.""" entry = catalog_by_id()["qwen3.8-27b"] choice = select_variant(entry, budget(8)) assert choice is not None assert not choice.zero_spill assert choice.reason_key == "smallest-fits-spilled" assert choice.variant.quant == entry.variants[-1].quant def test_frontier_model_refused_on_consumer_card_offered_on_big_ram(): """DeepSeek V4 Flash (161 GB at Q4): refused outright on a 32 GiB-RAM desktop; offered spilled on a 192 GiB-RAM workstation. The catalog carries frontier hardware honestly instead of hiding the model.""" entry = catalog_by_id()["deepseek-v4-flash"] assert select_variant(entry, budget(32, ram_gib=32)) is None big = select_variant(entry, budget(32, ram_gib=192)) assert big is not None and not big.zero_spill def test_selection_accounts_for_kv_not_just_weights(): """The zero-spill check prices weights + KV, not weights alone: give a machine exactly enough VRAM for the build's weights and the fit must come back spilled, not zero-spill.""" entry = catalog_by_id()["qwen3.8-27b"] build = entry.variants[0] exactly_weights = HardwareBudget( usable_vram_bytes=build.size_bytes + (100 << 20), total_device_bytes=build.size_bytes + (100 << 20), ram_available_bytes=64 * GIB) choice = select_variant(entry, exactly_weights) assert choice is not None assert not choice.zero_spill, "KV cost ignored — weights alone can't zero-spill" def test_target_never_degrades_below_floor_choice(): """The target preference may only IMPROVE the window, never the floor guarantees: whenever the old floor rule found a zero-spill pick, the new rule also finds one (possibly a smaller quant, never spill).""" for entry in CATALOG: for vram in (8, 12, 16, 24, 32, 48, 96): choice = select_variant(entry, budget(vram, ram_gib=256)) if choice is None: continue # Rule 2: whatever was chosen zero-spill must genuinely clear # the floor (the selector's own invariant, re-checked). if choice.zero_spill: assert choice.reason_key in ("best-large-window", "best-fits") def test_find_entry_for_model_resolves_split_ids(): hit = find_entry_for_model("DeepSeek-V4-Flash-0731-UD-Q4_K_XL") assert hit is not None entry, variant = hit assert entry.id == "deepseek-v4-flash" assert variant.quant == "UD-Q4_K_XL" def test_catalog_and_preset_agree_on_identical_model_facts(tmp_path, monkeypatch): from types import SimpleNamespace from hermes_cli.local_runtime import bootstrap, catalog, presets from hermes_cli.local_runtime.context_policy import RUNTIME_OVERHEAD_BYTES, ub_logits_bytes from hermes_cli.local_runtime.estimator import ctx_bytes from hermes_cli.web_routers.local_models import _catalog_row monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr("hermes_cli.web_routers.local_models._engine_too_old", lambda tag: False) for entry in catalog.CATALOG: variant = entry.variants[0] profile = entry.profile(variant) path = tmp_path / f"{variant.model_id}.gguf" monkeypatch.setattr(presets, "read_gguf_header", lambda p: SimpleNamespace(sampling_defaults={})) monkeypatch.setattr(presets, "profile_from_gguf", lambda h: profile) if entry.mmproj: asset = bootstrap.assets_dir() / entry.mmproj.local_name asset.parent.mkdir(parents=True, exist_ok=True) asset.touch() for vram in (16, 24, 32, 48): for uma in (False, True): machine = HardwareBudget(int(vram * GIB * 0.8), vram * GIB, 0 if uma else 32 * GIB, uma) row = _catalog_row(entry, machine, None, None, set()) preset = presets.preset_for_model(path, machine, set()) assert row["fits"] == (preset.refusal is None) if preset.refusal: continue assert row["start_window"] == preset.window assert row["spilled"] == preset.spilled overhead = (RUNTIME_OVERHEAD_BYTES + (entry.mmproj.size_bytes if entry.mmproj else 0) + ub_logits_bytes(profile.n_vocab, mtp_capable=entry.mtp, mtp_prefill=preset.keys.get("ubatch-size") == "2048" and entry.mtp)) need = profile.weights_bytes + ctx_bytes(profile, preset.window) + overhead assert preset.spilled == (need > machine.usable_vram_bytes) assert need <= machine.usable_vram_bytes + machine.ram_available_bytes def test_hybrid_long_context_stays_cheap(): """The reason Nemotron/Qwen3.6 headline the catalog: their priced 64K-floor KV must be a small fraction of a dense model's.""" from hermes_cli.local_runtime.catalog import FLOOR from hermes_cli.local_runtime.estimator import ctx_bytes from hermes_cli.local_runtime.estimator import LayerKind, ModelProfile hybrid = catalog_by_id()["qwen3.6-35b-a3b"] hybrid_profile = hybrid.profile(hybrid.variants[-1]) # A fully-dense profile of the same layer count and per-layer cost: # the contract is about LAYER ECONOMICS (recurrent layers pay no # per-token KV), not about any particular catalog entry. n_layers = len(hybrid_profile.layers) dense_profile = ModelProfile( name="synthetic-dense", weights_bytes=hybrid_profile.weights_bytes, embd_table_bytes=0, n_ctx_train=hybrid.n_ctx_train, layers=[(LayerKind.FULL, hybrid.per_layer_f16)] * n_layers) dense_kv = ctx_bytes(dense_profile, FLOOR) hybrid_kv = ctx_bytes(hybrid_profile, FLOOR) # The contract is structural: recurrent layers pay no per-token KV, # so the hybrid's KV must track its full-attention share (x kv_scale # for MTP's draft context), not its total layer count. full = sum(1 for kind, _ in hybrid_profile.layers if kind == LayerKind.FULL) expected = dense_kv * full / n_layers * hybrid_profile.kv_scale assert hybrid_kv < dense_kv, "hybrid must be cheaper than dense" assert abs(hybrid_kv - expected) / expected < 0.25, ( f"hybrid KV ({hybrid_kv:,}) should track its full-attention share " f"(expected ~{expected:,.0f})")