From 97ca41ce95a815f798c4fe6ae6643180ea891382 Mon Sep 17 00:00:00 2001 From: Laurent-Philippe Albou <24249870+lpalbou@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:41:57 +0100 Subject: [PATCH 1/5] ADR 0010: model residency failsafe (fit check, locks, LRU eviction of unlocked models, typed refusal with suggestions) --- .../0010-model-residency-fit-and-eviction.md | 66 +++++++++++++++++++ docs/adr/README.md | 1 + 2 files changed, 67 insertions(+) create mode 100644 docs/adr/0010-model-residency-fit-and-eviction.md diff --git a/docs/adr/0010-model-residency-fit-and-eviction.md b/docs/adr/0010-model-residency-fit-and-eviction.md new file mode 100644 index 00000000..ea54431a --- /dev/null +++ b/docs/adr/0010-model-residency-fit-and-eviction.md @@ -0,0 +1,66 @@ +# ADR 0010: Model residency failsafe — fit before load, evict unlocked LRU, refuse with a typed error + +Status: Accepted (2026-10-29). + +## Context + +Loading a model must never crash the gateway. On the Linux test server (one 16 GB GPU, 26 GB RAM, two +accounts' runtimes in one process) a load used whatever memory it could get: the llama.cpp context +ladder retried smaller contexts, then fell back to the CPU, and a model that did not fit next to the +resident ones either squeezed them, filled RAM, or failed deep inside llama.cpp with no sentence a +person can act on. Memory was never *managed*: nothing chose what to free. + +Core already owns the three facts this needs (ADR 0008): which instances hold which weights in this +process (`hf_residency`, `mlx_residency`), who claims them (`process_residency` claimants: runtime +pools, locks, builds), and how big a load is (`utils.kv_geometry`: weights + KV at a context). + +## Decision + +Core owns the failsafe, in `process_residency`, because the decision needs every holder in the +process, every claim, and the estimate; Runtime and Gateway only declare locks and surface the outcome. + +1. **One fit check, one place.** Before a provider allocates (`HuggingFaceProvider._load_gguf_model` + first; other in-process lanes adopt the same call), Core computes the need = file weights + KV at + the smallest context it would accept (the device rule of `docs/memory-management.md` then sizes the + context in what is left) + the reserve (max(1 GiB, 10% of the device)), and compares it with the + memory free *now* on the device the load targets (CUDA: `torch.cuda.mem_get_info`; CPU lane: + available RAM). One function: `process_residency.fit_check(...)`. +2. **What locks a model (nothing else does):** an admin's "Keep loaded" pin (the runtime's + `lock_model_residency`, the Models page toggle), and in-flight use (a generation running on an + instance of it). Pools, configured defaults, sessions and prompt caches do not lock. +3. **Last used, per model:** the newest `last_used_at` over the model's instances (stamped when a + generation begins and ends; the load time until the first call). +4. **Eviction order:** when the need does not fit, evict resident in-process models that are not + locked and not the one being loaded, least recently used first, one at a time (process-wide + eject: every holder of that model), re-measuring free memory after each, until it fits or none is + evictable. Locked models are never evicted by this process. An evicted model reloads on its next + call (the eject contract), so a runtime pool keeps working. +5. **The typed refusal:** if it still does not fit, the load fails with + `ModelDoesNotFitError(ProviderError)` carrying `model`, `needed_bytes`, `free_bytes`, + `device`, `locked` (the models kept, in words), `evicted`, and `suggestions` (the catalog's + smaller artifacts of the same model: smaller quant, smaller size, that fit `free + evictable`), + and the sentence: "MODEL needs 13.7 GB but only 9.2 GB is free on the GPU (kept: X, locked). + Choose a smaller model, for example: A (5.6 GB), B (4.1 GB), or unlock X." Nothing is allocated. +6. **Broadcast:** Runtime records the error type and its fields on the failed LLM_CALL / load in the + ledger (`error.code = "model_does_not_fit"`); the Gateway shows the sentence as a console toast and + on the Models page, and thin clients' chat renders the same sentence (the kit's error sentence). +7. **Multi-account runtimes:** a model held by several accounts' runtimes is one resident model (the + claims of all owners count once); it is locked if ANY owner locks it or any instance of it is in + use; evicting it ejects every holder (AbstractRuntime 0.15.x: dropped copies are released by their + owner, unbuilt defaults claim nothing). The refusal names locks by owner label ("the runtime of + account default/bob"), never by id (ADR 0043 of the framework). + +## Consequences + +- A load either fits (possibly after evicting idle unlocked models) or fails before allocating with a + sentence and alternatives; the CPU fallback is subject to the same check against RAM. +- Eviction is visible: each evicted model is logged and reported in `evicted` of the load result. +- Estimates are estimates (labelled `kv_cache_estimated`): the context ladder and its probe still + guard the real allocation; the fit check only prevents the loads that cannot fit. + +## Enforcement + +Tests in `tests/providers/test_residency_fit_and_evict_unit.py` go red when a locked or in-use model +is evicted, when the order is not LRU, when the refusal lacks its numbers or suggestions, or when a +load allocates without the check. A real-weights proof on the Linux test server (lock A, refuse B; +unlock A, B evicts A) is recorded in the PR. diff --git a/docs/adr/README.md b/docs/adr/README.md index db5d3ee5..4ceea898 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,6 +18,7 @@ that future work must respect. | [0007](0007-durable-memory-bloc-cache-binding.md) | Accepted | Durable memory bloc cache binding | Defines exact local bloc artifact binding across Python, endpoint, gateway, and provider hooks. | | [0008](0008-provider-owned-model-residency-truth.md) | Accepted | Provider-owned model residency truth | Requires model loaded-state truth to come from provider/capability contracts. | | [0009](0009-model-handle-fidelity.md) | Accepted | Model handle fidelity | Forbids loading a different model artifact than the handle names; amends 0001. | +| [0010](0010-model-residency-fit-and-eviction.md) | Accepted | Model residency fit and eviction | A load checks weights + KV against free memory, evicts unlocked LRU models, else refuses with a typed error and smaller suggestions. | ## How To Use This Set From 0aa667f6c7265d870d7bf4d3e6e9284f8a87dd57 Mon Sep 17 00:00:00 2001 From: Laurent-Philippe Albou <24249870+lpalbou@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:42:56 +0100 Subject: [PATCH 2/5] WIP failsafe: last-used stamps, registration time, ModelDoesNotFitError --- abstractcore/exceptions/__init__.py | 28 +++++++++++++++++++++ abstractcore/providers/generation_cancel.py | 5 ++++ abstractcore/providers/hf_residency.py | 4 ++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/abstractcore/exceptions/__init__.py b/abstractcore/exceptions/__init__.py index 68ec1edc..3a6af781 100644 --- a/abstractcore/exceptions/__init__.py +++ b/abstractcore/exceptions/__init__.py @@ -97,6 +97,33 @@ class InvalidRequestError(ProviderError): pass +class ModelDoesNotFitError(ProviderError): + """A model was asked to load, and its weights + KV cache do not fit the memory + that is free, even after evicting every resident model that is not locked + (core ADR 0010). Nothing was allocated. Fields say by how much and what to do: + `needed_bytes`, `free_bytes`, `device`, `kept` (locked / in-use models, in + words), `evicted`, `suggestions` (smaller artifacts of the same model).""" + + code = "model_does_not_fit" + + def __init__(self, message: str, *, model: "str | None" = None, needed_bytes: "int | None" = None, + free_bytes: "int | None" = None, device: "str | None" = None, kept: "list | None" = None, + evicted: "list | None" = None, suggestions: "list | None" = None): + super().__init__(message) + self.model = model + self.needed_bytes = needed_bytes + self.free_bytes = free_bytes + self.device = device + self.kept = list(kept or []) + self.evicted = list(evicted or []) + self.suggestions = list(suggestions or []) + + def as_dict(self) -> dict: + return {"code": self.code, "message": str(self), "model": self.model, "needed_bytes": self.needed_bytes, + "free_bytes": self.free_bytes, "device": self.device, "kept": self.kept, + "evicted": self.evicted, "suggestions": self.suggestions} + + class ContextWindowExceededError(InvalidRequestError): """The request needs more tokens than the context the model is loaded with. @@ -253,6 +280,7 @@ def format_provider_error(provider: str, reason: str) -> str: # Export all exceptions for easy importing __all__ = [ + 'ModelDoesNotFitError', 'ContextWindowExceededError', 'AbstractCoreError', 'ProviderError', diff --git a/abstractcore/providers/generation_cancel.py b/abstractcore/providers/generation_cancel.py index 56b04d41..8cc0977d 100644 --- a/abstractcore/providers/generation_cancel.py +++ b/abstractcore/providers/generation_cancel.py @@ -260,6 +260,9 @@ class InflightGenerations: def __init__(self) -> None: self._lock = threading.Lock() self._calls: "_weakref.WeakSet[_InflightCall]" = _weakref.WeakSet() + # Wall-clock time a generation last began or ended on this instance: the + # "least recently used" of the residency failsafe (core ADR 0010). + self.last_used_at: Optional[float] = None def begin(self, event: Optional[threading.Event]) -> Optional[_InflightCall]: if event is None: @@ -267,6 +270,7 @@ def begin(self, event: Optional[threading.Event]) -> Optional[_InflightCall]: call = _InflightCall(event) with self._lock: self._calls.add(call) + self.last_used_at = _time.time() return call def end(self, call: Optional[_InflightCall]) -> None: @@ -275,6 +279,7 @@ def end(self, call: Optional[_InflightCall]) -> None: call.done.set() with self._lock: self._calls.discard(call) + self.last_used_at = _time.time() def active(self) -> int: with self._lock: diff --git a/abstractcore/providers/hf_residency.py b/abstractcore/providers/hf_residency.py index 5c4d59e7..c7550181 100644 --- a/abstractcore/providers/hf_residency.py +++ b/abstractcore/providers/hf_residency.py @@ -60,7 +60,9 @@ def register_provider(provider: Any) -> None: try: with _HF_PROVIDERS_LOCK: _HF_PROVIDERS.add(provider) - except TypeError: + if getattr(provider, "_residency_registered_at", None) is None: + provider._residency_registered_at = time.time() + except (TypeError, AttributeError): pass From 631ef7785fdbcb273e8826176e1f844b5726730f Mon Sep 17 00:00:00 2001 From: Laurent-Philippe Albou <24249870+lpalbou@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:55:17 +0100 Subject: [PATCH 3/5] Residency failsafe (ADR 0010): fit check before the GGUF allocation, LRU eviction of unlocked idle models, ModelDoesNotFitError with numbers and smaller suggestions; CPU fallback checked against RAM --- CHANGELOG.md | 12 ++ .../providers/huggingface_provider.py | 69 ++++++++ abstractcore/providers/process_residency.py | 167 ++++++++++++++++++ .../test_residency_fit_and_evict_unit.py | 141 +++++++++++++++ 4 files changed, 389 insertions(+) create mode 100644 tests/providers/test_residency_fit_and_evict_unit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 021b7871..714c2512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Model residency failsafe (ADR 0010): a load never fills the card.** Before llama.cpp allocates (one full-offload + CUDA card), the GGUF load checks weights + KV at its smallest context + the reserve against the free device + memory; when it does not fit, resident models that are neither locked (a "Keep loaded" pin) nor in use are + evicted, least recently used first, one at a time (`process_residency.make_room`); if it still does not fit, the + load fails before allocating with `ModelDoesNotFitError` (`needed_bytes`, `free_bytes`, `kept`, `evicted`, + `suggestions`: smaller artifacts of the same model from the catalog) and a sentence ("X needs 13.7 GB but only + 9.2 GB is free on the GPU (kept: Y (locked by the runtime of account default/admin)). Choose a smaller model, for + example ..."). The CPU fallback is checked against available RAM the same way. `InflightGenerations.last_used_at` + and `_residency_registered_at` give the order. + ## [2.31.1] - 2026-10-28 ### Added diff --git a/abstractcore/providers/huggingface_provider.py b/abstractcore/providers/huggingface_provider.py index a976ec66..2b2084b6 100644 --- a/abstractcore/providers/huggingface_provider.py +++ b/abstractcore/providers/huggingface_provider.py @@ -79,6 +79,7 @@ def _module_available(name: str) -> bool: GenerationCancelledError, InvalidRequestError, ModelArtifactMismatchError, + ModelDoesNotFitError, ModelNotFoundError, ProviderError, format_model_error, @@ -3304,6 +3305,60 @@ def _gguf_reuse_live_context(self, prefix_len: int) -> bool: _GGUF_RESERVE_FLOOR_BYTES = 1 << 30 _GGUF_RESERVE_FRACTION = 0.10 + def _gguf_ensure_cpu_fit(self, model_path: Any, n_ctx: int) -> None: + """Raise `ModelDoesNotFitError` (device "RAM") when the available RAM cannot + hold the weights + KV at `n_ctx` + 1 GiB. Unknown figures: no refusal.""" + try: + from ..utils.memory import get_memory_snapshot + from ..utils.kv_geometry import kv_cache_bytes + from .hf_residency import gguf_file_geometry + + avail = ((get_memory_snapshot() or {}).get("ram") or {}).get("available_bytes") + weights_b = _gguf_total_weight_bytes(model_path) + kv = kv_cache_bytes(gguf_file_geometry(model_path), int(n_ctx), pad_v=True) + if not (isinstance(avail, int) and weights_b and kv): + return + need = int(weights_b) + int(kv) + (1 << 30) + except Exception: + return + if need <= int(avail): + return + from . import process_residency + + process_residency.ensure_fit(str(self.model), need, lambda: int( + ((get_memory_snapshot() or {}).get("ram") or {}).get("available_bytes") or 0), + provider="huggingface", device="RAM (CPU fallback)") + + def _gguf_ensure_fit(self, model_path: Any, min_ctx: int) -> Optional[Dict[str, Any]]: + """ADR 0010 on one full-offload CUDA card (the device rule's scope): need = + file weights + KV at `min_ctx` + reserve. None when the check does not + apply (CPU, partial offload, several cards, Metal, unknown geometry).""" + try: + if not (isinstance(self.n_gpu_layers, int) and self.n_gpu_layers < 0): + return None + import torch + + if not torch.cuda.is_available() or int(torch.cuda.device_count()) != 1: + return None + from ..utils.kv_geometry import kv_cache_bytes + from .hf_residency import gguf_file_geometry + + weights_b = _gguf_total_weight_bytes(model_path) + geom = gguf_file_geometry(model_path) + total_b = int(torch.cuda.mem_get_info()[1]) + if not weights_b or not geom: + return None + reserve = max(self._GGUF_RESERVE_FLOOR_BYTES, int(total_b * self._GGUF_RESERVE_FRACTION)) + need = int(weights_b) + int(kv_cache_bytes(geom, int(min_ctx), pad_v=True) or 0) + reserve + except Exception: + return None + from . import process_residency + + def _free() -> int: + return int(torch.cuda.mem_get_info()[0]) + + return process_residency.ensure_fit(str(self.model), need, _free, provider="huggingface", device="GPU") + def _gguf_device_fit_rungs(self, model_path: Any, candidate_ctxs: List[int]) -> List[int]: """`candidate_ctxs` without the rungs whose KV cache does not fit the device under the GGUF context rule (above). Unchanged when the device @@ -7509,6 +7564,14 @@ def _load_gguf_model(self): if requested_n_ctx_i <= 0: requested_n_ctx_i = 16384 + # THE FIT CHECK (core ADR 0010), before anything is allocated: the weights + # and the KV of the smallest context this load accepts must fit the free + # device memory, after evicting resident models that are not locked + # (least recently used first); else ModelDoesNotFitError, with numbers and + # smaller suggestions. Never a load that fills the card or the RAM. + _min_ctx = requested_n_ctx_i if getattr(self, "_user_provided_max_tokens", False) else min(4096, requested_n_ctx_i) + self._gguf_fit_report = self._gguf_ensure_fit(model_path, _min_ctx) + if getattr(self, "_user_provided_max_tokens", False): candidate_ctxs = [requested_n_ctx_i] else: @@ -7583,6 +7646,10 @@ def _load_gguf_model(self): is_metal_host = platform.system().lower() == "darwin" and platform.machine().lower() == "arm64" cpu_retry_now = is_metal_host or n_ctx_i == candidate_ctxs[-1] if isinstance(self.n_gpu_layers, int) and self.n_gpu_layers != 0 and cpu_retry_now: + # ADR 0010: the CPU fallback puts weights AND the KV cache in RAM; + # refuse it with the numbers when RAM cannot hold them, instead of + # letting the kernel kill the process. + self._gguf_ensure_cpu_fit(model_path, int(n_ctx_i)) try: self.logger.warning( f"GGUF load failed with n_gpu_layers={self.n_gpu_layers}; retrying with CPU (n_gpu_layers=0). Error: {e}" @@ -7771,6 +7838,8 @@ def _load_gguf_model(self): except Exception: pass + except ModelDoesNotFitError: + raise # ADR 0010: the typed refusal reaches the caller as itself except ModelArtifactMismatchError: # ADR 0009 Enforcement: an artifact-fidelity refusal must reach the caller # as itself. Wrapping it in RuntimeError here made the contract type diff --git a/abstractcore/providers/process_residency.py b/abstractcore/providers/process_residency.py index 70ffb8fe..a1e67efd 100644 --- a/abstractcore/providers/process_residency.py +++ b/abstractcore/providers/process_residency.py @@ -249,3 +249,170 @@ def eject_unclaimed(provider: str, model: str, *, task: Optional[str] = None, pa "locked": any(bool(c.get("locked")) for c in claims), "reason": f"{model} is still in use by {', '.join(owners)}; not ejected"} return eject(backend, model, reason=reason) + + +# --------------------------------------------------------------------------- +# The residency failsafe (core ADR 0010): a load checks weights + KV against +# the free memory, evicts resident models that are not locked (least recently +# used first, one at a time), and otherwise refuses with a typed error. +# --------------------------------------------------------------------------- +_GB = 1000 ** 3 + + +def _fmt_gb(n: Optional[int]) -> str: + return "unknown" if n is None else f"{n / _GB:.1f} GB" + + +def _hf_holder_state(holder: Any) -> Dict[str, Any]: + from .hf_residency import _holder_row + + row = _holder_row(holder) + inflight = 0 + last_used = getattr(holder, "_residency_registered_at", None) + try: + registry = holder._inflight_generations() + inflight = int(registry.active()) + if getattr(registry, "last_used_at", None): + last_used = max(float(last_used or 0.0), float(registry.last_used_at)) + except Exception: # noqa: BLE001 - a holder without the registry is never "in use" by it + pass + return {"held_bytes": int(row.get("held_bytes") or 0), "inflight": inflight, "last_used_at": last_used} + + +def resident_model_states() -> List[Dict[str, Any]]: + """Every in-process HuggingFace model resident now, with what the failsafe + decides on: `held_bytes`, `last_used_at` (newest over its instances), + `in_use` (a generation running on any instance), `locked_by` (the readable + owners whose claim locks it). MLX and embeddings are not evicted by the + failsafe yet (ADR 0010: adopted lane by lane).""" + if not _backend_loaded("huggingface"): + return [] + from .hf_residency import _holder_alive, registered_providers + + by_model: Dict[str, Dict[str, Any]] = {} + for holder in registered_providers(): + try: + if not _holder_alive(holder): + continue + name = str(getattr(holder, "model", "") or "").strip() + if not name: + continue + st = _hf_holder_state(holder) + except Exception: # noqa: BLE001 + continue + cur = by_model.setdefault(name, {"backend": "huggingface", "provider": "huggingface", "model": name, + "held_bytes": 0, "last_used_at": None, "in_use": False, "holders": 0}) + cur["holders"] += 1 + cur["held_bytes"] += st["held_bytes"] + cur["in_use"] = cur["in_use"] or st["inflight"] > 0 + if st["last_used_at"] is not None: + cur["last_used_at"] = max(float(cur["last_used_at"] or 0.0), float(st["last_used_at"])) + for state in by_model.values(): + claims = claims_for("huggingface", state["model"]) + state["locked_by"] = sorted({str(c.get("owner_label") or "another owner in this process") + for c in claims if c.get("locked")}) + return list(by_model.values()) + + +def make_room(needed_bytes: int, free_bytes: Any, *, provider: Optional[str] = None, model: Optional[str] = None, + reason: str = "make_room") -> Dict[str, Any]: + """Evict resident models that are neither locked nor in use, least recently + used first, one at a time, until `free_bytes()` reaches `needed_bytes` or + none is evictable. Never evicts `model` itself. Returns `{"fits", "free_bytes", + "evicted": [{model, freed_bytes}], "kept": [readable reasons]}`.""" + evicted: List[Dict[str, Any]] = [] + with _RESIDENCY_LOCK: + free = int(free_bytes()) + while free < int(needed_bytes): + states = [s for s in resident_model_states() + if not (model and model_matches(s["model"], None, model))] + candidates = [s for s in states if not s["locked_by"] and not s["in_use"]] + if not candidates: + break + victim = min(candidates, key=lambda s: float(s["last_used_at"] or 0.0)) + report = eject(victim["backend"], victim["model"], reason=reason) + after = int(free_bytes()) + evicted.append({"model": victim["model"], "freed_bytes": max(0, after - free), + "ok": bool(report.get("ok", True))}) + if after <= free and not report.get("ok", True): + break # it did not free anything: stop rather than loop + free = after + kept = [] + for s in resident_model_states(): + if model and model_matches(s["model"], None, model): + continue + if s["locked_by"]: + kept.append(f"{s['model']} (locked by {', '.join(s['locked_by'])})") + elif s["in_use"]: + kept.append(f"{s['model']} (in use)") + return {"fits": free >= int(needed_bytes), "free_bytes": free, "evicted": evicted, "kept": kept} + + +def smaller_suggestions(model: str, budget_bytes: int, *, provider: str = "huggingface", + limit: int = 3) -> List[Dict[str, Any]]: + """Catalog artifacts of the same model (or its family) on `provider` whose + download fits `budget_bytes` with room for a KV cache, largest first.""" + try: + from ..config.model_catalog import load_seed + + rows = list(load_seed().get("rows") or []) + except Exception: # noqa: BLE001 + return [] + name = str(model or "").lower() + same = [r for r in rows if any(str(a.get("artifact") or "").lower() == name for a in r.get("artifacts") or [])] + if not same: + same = [r for r in rows if str(r.get("id") or "").lower() and str(r.get("id")).lower() in name.replace("_", "-")] + families = {str(r.get("family") or "") for r in same if r.get("family")} + pool = same + [r for r in rows if r not in same and str(r.get("family") or "") in families] + out: List[Dict[str, Any]] = [] + for r in pool: + for a in r.get("artifacts") or []: + size = a.get("download_bytes") + if str(a.get("provider") or "") != provider or not isinstance(size, int): + continue + if str(a.get("artifact") or "").lower() == name: + continue + if size * 1.15 + (1 << 30) > budget_bytes: + continue + out.append({"model": r.get("display_name") or r.get("id"), "artifact": a.get("artifact"), + "quant": a.get("quant"), "download_bytes": size}) + out.sort(key=lambda x: -int(x["download_bytes"])) + seen, uniq = set(), [] + for item in out: + if item["artifact"] in seen: + continue + seen.add(item["artifact"]) + uniq.append(item) + return uniq[:limit] + + +def ensure_fit(model: str, needed_bytes: int, free_bytes: Any, *, provider: str = "huggingface", + device: str = "GPU") -> Dict[str, Any]: + """THE fit check (ADR 0010). Evicts unlocked LRU models when needed; raises + `ModelDoesNotFitError` with the sentence, numbers and smaller suggestions when + the load still cannot fit. Returns the make-room report when it fits.""" + report = make_room(needed_bytes, free_bytes, provider=provider, model=model, reason=f"room for {model}") + for ev in report["evicted"]: + import logging + + logging.getLogger("abstractcore.residency").warning( + f"evicted {ev['model']} (least recently used, not locked) to make room for {model}") + if report["fits"]: + return report + from ..exceptions import ModelDoesNotFitError + + free = int(report["free_bytes"]) + suggestions = smaller_suggestions(model, free, provider=provider) + kept = report["kept"] + sentence = f"{model} needs {_fmt_gb(needed_bytes)} but only {_fmt_gb(free)} is free on the {device}" + if kept: + sentence += f" (kept: {'; '.join(kept)})" + sentence += ". Choose a smaller model" + if suggestions: + sentence += ", for example " + ", ".join( + f"{s['artifact']} ({_fmt_gb(s['download_bytes'])})" for s in suggestions) + if any("locked" in k for k in kept): + sentence += ", or unlock a kept model on the Models page" + sentence += "." + raise ModelDoesNotFitError(sentence, model=model, needed_bytes=int(needed_bytes), free_bytes=free, + device=device, kept=kept, evicted=report["evicted"], suggestions=suggestions) diff --git a/tests/providers/test_residency_fit_and_evict_unit.py b/tests/providers/test_residency_fit_and_evict_unit.py new file mode 100644 index 00000000..2b09619c --- /dev/null +++ b/tests/providers/test_residency_fit_and_evict_unit.py @@ -0,0 +1,141 @@ +"""The residency failsafe (core ADR 0010): fit before load, evict unlocked LRU, +refuse with a typed error and smaller suggestions. Fakes only: no weights.""" +from __future__ import annotations + +import gc +import time +from typing import Any, Dict, List + +import pytest + +import abstractcore.providers.huggingface_provider # noqa: F401 - the HF backend is "loaded" +from abstractcore.exceptions import ModelDoesNotFitError +from abstractcore.providers import hf_residency, process_residency as pr +from abstractcore.providers.generation_cancel import InflightGenerations + +GB = 1000 ** 3 + + +class _Holder: + def __init__(self, model: str, size: int, last_used: float): + self.model = model + self.llm = object() + self.size = size + self._reg = InflightGenerations() + self._reg.last_used_at = last_used + + def _inflight_generations(self): + return self._reg + + +class _Card: + """A device whose free memory grows when a holder is ejected.""" + + def __init__(self, total: int, holders: List[_Holder]): + self.total = total + self.holders = holders + + def free(self) -> int: + return self.total - sum(h.size for h in self.holders if h.llm is not None) + + +class _Locker: + def __init__(self, model: str, label: str): + self.model, self.label = model, label + + def residency_claims(self): + return [{"provider": "huggingface", "model": self.model, "locked": True, "kind": "lock", + "owner": "x", "owner_label": self.label}] + + +@pytest.fixture +def card(monkeypatch): + gc.collect() + holders: List[_Holder] = [] + c = _Card(15 * GB, holders) + monkeypatch.setattr(hf_residency, "registered_providers", lambda: list(holders)) + monkeypatch.setattr(hf_residency, "_holder_alive", lambda h: h.llm is not None) + monkeypatch.setattr(hf_residency, "_holder_row", lambda h: {"held_bytes": h.size}) + ejected: List[str] = [] + + def fake_eject(backend, model, reason="eject"): + for h in holders: + if h.model == model: + h.llm = None + ejected.append(model) + return {"ok": True} + + monkeypatch.setattr(pr, "eject", fake_eject) + c.ejected = ejected + return c + + +def test_evicts_least_recently_used_first_and_only_what_is_needed(card): + now = time.time() + card.holders += [_Holder("old/A", 4 * GB, now - 300), _Holder("recent/B", 4 * GB, now - 10), + _Holder("older/C", 4 * GB, now - 600)] + out = pr.ensure_fit("new/D", 6 * GB, card.free) # free 3 GB -> one eviction of 4 GB is enough + assert card.ejected == ["older/C"], f"MUTANT: not LRU -> RED ({card.ejected})" + assert out["fits"] and [e["model"] for e in out["evicted"]] == ["older/C"] + + +def test_locked_and_in_use_models_are_never_evicted(card): + now = time.time() + a, b = _Holder("locked/A", 6 * GB, now - 900), _Holder("busy/B", 6 * GB, now - 800) + card.holders += [a, b] + lock = _Locker("locked/A", "the runtime of account default/admin") + pr.register_claimant(lock) + running = type("C", (), {})() # a generation running on B (held: a WeakSet) + b._reg._calls.add(running) + try: + with pytest.raises(ModelDoesNotFitError) as err: + pr.ensure_fit("big/E", 8 * GB, card.free) + finally: + pr.unregister_claimant(lock) + assert card.ejected == [], "MUTANT: lock / in-flight ignored -> evicted -> RED" + e = err.value + assert e.needed_bytes == 8 * GB and e.free_bytes == 3 * GB and e.device == "GPU" + msg = str(e) + assert "big/E needs 8.0 GB but only 3.0 GB is free on the GPU" in msg + assert "locked/A (locked by the runtime of account default/admin)" in msg and "busy/B (in use)" in msg + assert "unlock" in msg + + +def test_refusal_suggests_smaller_artifacts_of_the_same_model(card, monkeypatch): + import abstractcore.config.model_catalog as mc + + monkeypatch.setattr(mc, "load_seed", lambda: {"rows": [{ + "id": "qwen3.5-9b", "family": "qwen3.5", "display_name": "Qwen3.5 9B", "artifacts": [ + {"provider": "huggingface", "artifact": "unsloth/Qwen3.5-9B-GGUF:Q8_0", "quant": "q8_0", "download_bytes": 9 * GB}, + {"provider": "huggingface", "artifact": "unsloth/Qwen3.5-9B-GGUF:Q4_K_M", "quant": "q4_k_m", "download_bytes": int(5.6 * GB)}, + {"provider": "mlx", "artifact": "mlx/x", "quant": "4bit", "download_bytes": GB}]}, + {"id": "qwen3.5-4b", "family": "qwen3.5", "display_name": "Qwen3.5 4B", "artifacts": [ + {"provider": "huggingface", "artifact": "unsloth/Qwen3.5-4B-GGUF:Q4_K_M", "quant": "q4_k_m", "download_bytes": int(2.7 * GB)}]}]}) + card.holders.append(_Holder("pinned/A", 7 * GB, time.time())) + lock = _Locker("pinned/A", "the runtime of account default/admin") + pr.register_claimant(lock) + try: + with pytest.raises(ModelDoesNotFitError) as err: + pr.ensure_fit("unsloth/Qwen3.5-9B-GGUF:Q8_0", 11 * GB, card.free) + finally: + pr.unregister_claimant(lock) + arts = [s["artifact"] for s in err.value.suggestions] + assert arts == ["unsloth/Qwen3.5-9B-GGUF:Q4_K_M", "unsloth/Qwen3.5-4B-GGUF:Q4_K_M"], arts + assert "unsloth/Qwen3.5-9B-GGUF:Q4_K_M (5.6 GB)" in str(err.value) + + +def test_the_model_being_loaded_is_never_its_own_victim(card): + card.holders.append(_Holder("same/M", 12 * GB, 0.0)) + with pytest.raises(ModelDoesNotFitError): + pr.ensure_fit("same/M", 14 * GB, card.free) + assert card.ejected == [] + + +def test_the_gguf_load_runs_the_fit_check_before_allocating(monkeypatch): + """MUTANT: no call in `_load_gguf_model` -> llama.cpp allocates first -> RED.""" + import inspect + from abstractcore.providers.huggingface_provider import HuggingFaceProvider + + src = inspect.getsource(HuggingFaceProvider._load_gguf_model) + assert src.index("self._gguf_ensure_fit(") < src.index("llama_cls(**llama_kwargs)") + assert "self._gguf_ensure_cpu_fit(" in src From d289992513475695e25d74e078e50e67c3c515b8 Mon Sep 17 00:00:00 2001 From: Laurent-Philippe Albou <24249870+lpalbou@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:01:10 +0100 Subject: [PATCH 4/5] Suggestions: a fine-tune the catalog does not list finds its family by name --- abstractcore/providers/process_residency.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/abstractcore/providers/process_residency.py b/abstractcore/providers/process_residency.py index a1e67efd..df0224eb 100644 --- a/abstractcore/providers/process_residency.py +++ b/abstractcore/providers/process_residency.py @@ -363,6 +363,15 @@ def smaller_suggestions(model: str, budget_bytes: int, *, provider: str = "huggi if not same: same = [r for r in rows if str(r.get("id") or "").lower() and str(r.get("id")).lower() in name.replace("_", "-")] families = {str(r.get("family") or "") for r in same if r.get("family")} + if not families: + # A fine-tune or re-quant the catalog does not list ("mradermacher/Gemma-4-14B- + # A4B-Heretic-GGUF"): its family by name ("gemma4" in "gemma-4-14b-..."). + import re as _re + + flat = _re.sub(r"[^a-z0-9]", "", name) + families = {str(r.get("family")) for r in rows + if r.get("family") and len(_re.sub(r"[^a-z0-9]", "", str(r.get("family")))) >= 5 + and _re.sub(r"[^a-z0-9]", "", str(r.get("family")).lower()) in flat} pool = same + [r for r in rows if r not in same and str(r.get("family") or "") in families] out: List[Dict[str, Any]] = [] for r in pool: From 37ab65f256a0dc334643c9fa846c70dc9ef4e723 Mon Sep 17 00:00:00 2001 From: Laurent-Philippe Albou <24249870+lpalbou@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:14:27 +0100 Subject: [PATCH 5/5] Review: one GGUF load at a time (check + allocation), CUDA cache returned before measuring, in-use re-checked before each eviction, suggestions use the fit reserve, the CPU guard refuses without evicting and never on Metal --- .../providers/huggingface_provider.py | 20 +++++-- abstractcore/providers/process_residency.py | 52 ++++++++++++++++--- .../test_residency_fit_and_evict_unit.py | 14 ++++- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/abstractcore/providers/huggingface_provider.py b/abstractcore/providers/huggingface_provider.py index 2b2084b6..0663329f 100644 --- a/abstractcore/providers/huggingface_provider.py +++ b/abstractcore/providers/huggingface_provider.py @@ -3323,11 +3323,10 @@ def _gguf_ensure_cpu_fit(self, model_path: Any, n_ctx: int) -> None: return if need <= int(avail): return + # No eviction here: the resident models sit on the GPU, evicting them frees no RAM. from . import process_residency - process_residency.ensure_fit(str(self.model), need, lambda: int( - ((get_memory_snapshot() or {}).get("ram") or {}).get("available_bytes") or 0), - provider="huggingface", device="RAM (CPU fallback)") + process_residency.refuse_fit(str(self.model), need, int(avail), device="RAM (CPU fallback)", kept=[]) def _gguf_ensure_fit(self, model_path: Any, min_ctx: int) -> Optional[Dict[str, Any]]: """ADR 0010 on one full-offload CUDA card (the device rule's scope): need = @@ -3357,7 +3356,8 @@ def _gguf_ensure_fit(self, model_path: Any, min_ctx: int) -> Optional[Dict[str, def _free() -> int: return int(torch.cuda.mem_get_info()[0]) - return process_residency.ensure_fit(str(self.model), need, _free, provider="huggingface", device="GPU") + return process_residency.ensure_fit(str(self.model), need, _free, provider="huggingface", device="GPU", + reserve_bytes=reserve) def _gguf_device_fit_rungs(self, model_path: Any, candidate_ctxs: List[int]) -> List[int]: """`candidate_ctxs` without the rungs whose KV cache does not fit the @@ -7433,6 +7433,15 @@ def _normalize_to_cache_format(self, model_name: str) -> str: return f"models--{model_name}" def _load_gguf_model(self): + """Load the GGUF under the process load lock (ADR 0010): the fit check and the + allocation it admits are one step, so two loads cannot both pass the check + against the same free memory and then run out of it.""" + from .process_residency import load_lock + + with load_lock(): + return self._load_gguf_model_unlocked() + + def _load_gguf_model_unlocked(self): """Load GGUF model using llama-cpp-python (cache-only, no downloading)""" import os try: @@ -7649,7 +7658,8 @@ def _load_gguf_model(self): # ADR 0010: the CPU fallback puts weights AND the KV cache in RAM; # refuse it with the numbers when RAM cannot hold them, instead of # letting the kernel kill the process. - self._gguf_ensure_cpu_fit(model_path, int(n_ctx_i)) + if not is_metal_host: # Metal retries every rung on the CPU: its ladder decides + self._gguf_ensure_cpu_fit(model_path, int(n_ctx_i)) try: self.logger.warning( f"GGUF load failed with n_gpu_layers={self.n_gpu_layers}; retrying with CPU (n_gpu_layers=0). Error: {e}" diff --git a/abstractcore/providers/process_residency.py b/abstractcore/providers/process_residency.py index df0224eb..92b27a1d 100644 --- a/abstractcore/providers/process_residency.py +++ b/abstractcore/providers/process_residency.py @@ -257,6 +257,23 @@ def eject_unclaimed(provider: str, model: str, *, task: Optional[str] = None, pa # used first, one at a time), and otherwise refuses with a typed error. # --------------------------------------------------------------------------- _GB = 1000 ** 3 +_LOAD_LOCK = _threading.Lock() + + +def load_lock() -> "_threading.Lock": + """One in-process model load at a time (ADR 0010): fit check + allocation.""" + return _LOAD_LOCK + + +def _release_torch_device_cache() -> None: + """Freed torch tensors stay in its CUDA cache, which the free-memory probe + counts as used: return them before measuring.""" + torch = sys.modules.get("torch") + try: + if torch is not None and torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: # noqa: BLE001 + pass def _fmt_gb(n: Optional[int]) -> str: @@ -330,7 +347,19 @@ def make_room(needed_bytes: int, free_bytes: Any, *, provider: Optional[str] = N if not candidates: break victim = min(candidates, key=lambda s: float(s["last_used_at"] or 0.0)) + # Re-checked right before the eject: a generation that started since is + # never cancelled for room (the next candidate is tried instead). + again = next((s for s in resident_model_states() if s["model"] == victim["model"]), None) + if again is None or again["in_use"] or again["locked_by"]: + if again is None: + continue + states_skip = victim["model"] + candidates = [c for c in candidates if c["model"] != states_skip] + if not candidates: + break + continue report = eject(victim["backend"], victim["model"], reason=reason) + _release_torch_device_cache() after = int(free_bytes()) evicted.append({"model": victim["model"], "freed_bytes": max(0, after - free), "ok": bool(report.get("ok", True))}) @@ -349,7 +378,7 @@ def make_room(needed_bytes: int, free_bytes: Any, *, provider: Optional[str] = N def smaller_suggestions(model: str, budget_bytes: int, *, provider: str = "huggingface", - limit: int = 3) -> List[Dict[str, Any]]: + limit: int = 3, reserve_bytes: int = 1 << 30) -> List[Dict[str, Any]]: """Catalog artifacts of the same model (or its family) on `provider` whose download fits `budget_bytes` with room for a KV cache, largest first.""" try: @@ -381,7 +410,8 @@ def smaller_suggestions(model: str, budget_bytes: int, *, provider: str = "huggi continue if str(a.get("artifact") or "").lower() == name: continue - if size * 1.15 + (1 << 30) > budget_bytes: + # The fit check's own arithmetic: weights + a small-context KV (~15%) + the reserve. + if size * 1.15 + int(reserve_bytes) > budget_bytes: continue out.append({"model": r.get("display_name") or r.get("id"), "artifact": a.get("artifact"), "quant": a.get("quant"), "download_bytes": size}) @@ -396,7 +426,7 @@ def smaller_suggestions(model: str, budget_bytes: int, *, provider: str = "huggi def ensure_fit(model: str, needed_bytes: int, free_bytes: Any, *, provider: str = "huggingface", - device: str = "GPU") -> Dict[str, Any]: + device: str = "GPU", reserve_bytes: int = 1 << 30) -> Dict[str, Any]: """THE fit check (ADR 0010). Evicts unlocked LRU models when needed; raises `ModelDoesNotFitError` with the sentence, numbers and smaller suggestions when the load still cannot fit. Returns the make-room report when it fits.""" @@ -408,11 +438,19 @@ def ensure_fit(model: str, needed_bytes: int, free_bytes: Any, *, provider: str f"evicted {ev['model']} (least recently used, not locked) to make room for {model}") if report["fits"]: return report + refuse_fit(model, needed_bytes, int(report["free_bytes"]), device=device, kept=report["kept"], + evicted=report["evicted"], provider=provider, reserve_bytes=reserve_bytes) + return report # pragma: no cover - refuse_fit raises + + +def refuse_fit(model: str, needed_bytes: int, free: int, *, device: str, kept: List[str], + evicted: Optional[List[Dict[str, Any]]] = None, provider: str = "huggingface", + reserve_bytes: int = 1 << 30) -> None: + """Raise the typed refusal (ADR 0010) with its sentence and suggestions.""" from ..exceptions import ModelDoesNotFitError - free = int(report["free_bytes"]) - suggestions = smaller_suggestions(model, free, provider=provider) - kept = report["kept"] + evicted = list(evicted or []) + suggestions = smaller_suggestions(model, free, provider=provider, reserve_bytes=reserve_bytes) sentence = f"{model} needs {_fmt_gb(needed_bytes)} but only {_fmt_gb(free)} is free on the {device}" if kept: sentence += f" (kept: {'; '.join(kept)})" @@ -424,4 +462,4 @@ def ensure_fit(model: str, needed_bytes: int, free_bytes: Any, *, provider: str sentence += ", or unlock a kept model on the Models page" sentence += "." raise ModelDoesNotFitError(sentence, model=model, needed_bytes=int(needed_bytes), free_bytes=free, - device=device, kept=kept, evicted=report["evicted"], suggestions=suggestions) + device=device, kept=kept, evicted=evicted, suggestions=suggestions) diff --git a/tests/providers/test_residency_fit_and_evict_unit.py b/tests/providers/test_residency_fit_and_evict_unit.py index 2b09619c..d767ece4 100644 --- a/tests/providers/test_residency_fit_and_evict_unit.py +++ b/tests/providers/test_residency_fit_and_evict_unit.py @@ -136,6 +136,18 @@ def test_the_gguf_load_runs_the_fit_check_before_allocating(monkeypatch): import inspect from abstractcore.providers.huggingface_provider import HuggingFaceProvider - src = inspect.getsource(HuggingFaceProvider._load_gguf_model) + src = inspect.getsource(HuggingFaceProvider._load_gguf_model_unlocked) assert src.index("self._gguf_ensure_fit(") < src.index("llama_cls(**llama_kwargs)") assert "self._gguf_ensure_cpu_fit(" in src + # one load at a time: the check and the allocation it admits under one lock (review) + assert "with load_lock():" in inspect.getsource(HuggingFaceProvider._load_gguf_model) + + +def test_suggestions_use_the_fit_reserve(monkeypatch): + import abstractcore.config.model_catalog as mc + + monkeypatch.setattr(mc, "load_seed", lambda: {"rows": [{"id": "fam-x", "family": "famx", "artifacts": [ + {"provider": "huggingface", "artifact": "org/fam-x:Q4", "download_bytes": int(4.3 * GB)}]}]}) + assert pr.smaller_suggestions("org/fam-x:Q8", int(6.1 * GB), reserve_bytes=int(1.6 * GB)) == [], ( + "MUTANT: a 1 GiB reserve -> suggests a model the check itself refuses -> RED") + assert pr.smaller_suggestions("org/fam-x:Q8", int(7 * GB), reserve_bytes=int(1.6 * GB))