Residency failsafe (ADR 0010): fit before load, evict unlocked LRU, typed refusal with suggestions

CHANGELOG.md

@@ -26,6 +26,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   values for keyword argument 'cancel_event'" (backlog 0861): the structured branch hands the event over once,
   under the provider's cancel kwarg.
 
+### Added
+
+- **Model residency failsafe (ADR 0010): a load never fills the card.** Before llama.cpp allocates (one full-offload
+  CUDA card), the GGUF load checks weights + KV at its smallest context + the reserve against the free device
+  memory; when it does not fit, resident models that are neither locked (a "Keep loaded" pin) nor in use are
+  evicted, least recently used first, one at a time (`process_residency.make_room`); if it still does not fit, the
+  load fails before allocating with `ModelDoesNotFitError` (`needed_bytes`, `free_bytes`, `kept`, `evicted`,
+  `suggestions`: smaller artifacts of the same model from the catalog) and a sentence ("X needs 13.7 GB but only
+  9.2 GB is free on the GPU (kept: Y (locked by the runtime of account default/admin)). Choose a smaller model, for
+  example ..."). The CPU fallback is checked against available RAM the same way. `InflightGenerations.last_used_at`
+  and `_residency_registered_at` give the order.
+
 ## [2.31.1] - 2026-10-28
 
 ### Added

abstractcore/exceptions/__init__.py

@@ -97,6 +97,33 @@ class InvalidRequestError(ProviderError):
     pass
 
 
+class ModelDoesNotFitError(ProviderError):
+    """A model was asked to load, and its weights + KV cache do not fit the memory
+    that is free, even after evicting every resident model that is not locked
+    (core ADR 0010). Nothing was allocated. Fields say by how much and what to do:
+    `needed_bytes`, `free_bytes`, `device`, `kept` (locked / in-use models, in
+    words), `evicted`, `suggestions` (smaller artifacts of the same model)."""
+
+    code = "model_does_not_fit"
+
+    def __init__(self, message: str, *, model: "str | None" = None, needed_bytes: "int | None" = None,
+                 free_bytes: "int | None" = None, device: "str | None" = None, kept: "list | None" = None,
+                 evicted: "list | None" = None, suggestions: "list | None" = None):
+        super().__init__(message)
+        self.model = model
+        self.needed_bytes = needed_bytes
+        self.free_bytes = free_bytes
+        self.device = device
+        self.kept = list(kept or [])
+        self.evicted = list(evicted or [])
+        self.suggestions = list(suggestions or [])
+
+    def as_dict(self) -> dict:
+        return {"code": self.code, "message": str(self), "model": self.model, "needed_bytes": self.needed_bytes,
+                "free_bytes": self.free_bytes, "device": self.device, "kept": self.kept,
+                "evicted": self.evicted, "suggestions": self.suggestions}
+
+
 class ContextWindowExceededError(InvalidRequestError):
     """The request needs more tokens than the context the model is loaded with.
 
@@ -253,6 +280,7 @@ def format_provider_error(provider: str, reason: str) -> str:
 
 # Export all exceptions for easy importing
 __all__ = [
+    'ModelDoesNotFitError',
     'ContextWindowExceededError',
     'AbstractCoreError',
     'ProviderError',

abstractcore/providers/generation_cancel.py

@@ -260,13 +260,17 @@ class InflightGenerations:
     def __init__(self) -> None:
         self._lock = threading.Lock()
         self._calls: "_weakref.WeakSet[_InflightCall]" = _weakref.WeakSet()
+        # Wall-clock time a generation last began or ended on this instance: the
+        # "least recently used" of the residency failsafe (core ADR 0010).
+        self.last_used_at: Optional[float] = None
 
     def begin(self, event: Optional[threading.Event]) -> Optional[_InflightCall]:
         if event is None:
             return None
         call = _InflightCall(event)
         with self._lock:
             self._calls.add(call)
+            self.last_used_at = _time.time()
         return call
 
     def end(self, call: Optional[_InflightCall]) -> None:
@@ -275,6 +279,7 @@ def end(self, call: Optional[_InflightCall]) -> None:
         call.done.set()
         with self._lock:
             self._calls.discard(call)
+            self.last_used_at = _time.time()
 
     def active(self) -> int:
         with self._lock:

abstractcore/providers/hf_residency.py

@@ -60,7 +60,9 @@ def register_provider(provider: Any) -> None:
     try:
         with _HF_PROVIDERS_LOCK:
             _HF_PROVIDERS.add(provider)
-    except TypeError:
+        if getattr(provider, "_residency_registered_at", None) is None:
+            provider._residency_registered_at = time.time()
+    except (TypeError, AttributeError):
         pass
 
 

abstractcore/providers/huggingface_provider.py

@@ -79,6 +79,7 @@ def _module_available(name: str) -> bool:
     GenerationCancelledError,
     InvalidRequestError,
     ModelArtifactMismatchError,
+    ModelDoesNotFitError,
     ModelNotFoundError,
     ProviderError,
     format_model_error,
@@ -3304,6 +3305,60 @@ def _gguf_reuse_live_context(self, prefix_len: int) -> bool:
     _GGUF_RESERVE_FLOOR_BYTES = 1 << 30
     _GGUF_RESERVE_FRACTION = 0.10
 
+    def _gguf_ensure_cpu_fit(self, model_path: Any, n_ctx: int) -> None:
+        """Raise `ModelDoesNotFitError` (device "RAM") when the available RAM cannot
+        hold the weights + KV at `n_ctx` + 1 GiB. Unknown figures: no refusal."""
+        try:
+            from ..utils.memory import get_memory_snapshot
+            from ..utils.kv_geometry import kv_cache_bytes
+            from .hf_residency import gguf_file_geometry
+
+            avail = ((get_memory_snapshot() or {}).get("ram") or {}).get("available_bytes")
+            weights_b = _gguf_total_weight_bytes(model_path)
+            kv = kv_cache_bytes(gguf_file_geometry(model_path), int(n_ctx), pad_v=True)
+            if not (isinstance(avail, int) and weights_b and kv):
+                return
+            need = int(weights_b) + int(kv) + (1 << 30)
+        except Exception:
+            return
+        if need <= int(avail):
+            return
+        # No eviction here: the resident models sit on the GPU, evicting them frees no RAM.
+        from . import process_residency
+
+        process_residency.refuse_fit(str(self.model), need, int(avail), device="RAM (CPU fallback)", kept=[])
+
+    def _gguf_ensure_fit(self, model_path: Any, min_ctx: int) -> Optional[Dict[str, Any]]:
+        """ADR 0010 on one full-offload CUDA card (the device rule's scope): need =
+        file weights + KV at `min_ctx` + reserve. None when the check does not
+        apply (CPU, partial offload, several cards, Metal, unknown geometry)."""
+        try:
+            if not (isinstance(self.n_gpu_layers, int) and self.n_gpu_layers < 0):
+                return None
+            import torch
+
+            if not torch.cuda.is_available() or int(torch.cuda.device_count()) != 1:
+                return None
+            from ..utils.kv_geometry import kv_cache_bytes
+            from .hf_residency import gguf_file_geometry
+
+            weights_b = _gguf_total_weight_bytes(model_path)
+            geom = gguf_file_geometry(model_path)
+            total_b = int(torch.cuda.mem_get_info()[1])
+            if not weights_b or not geom:
+                return None
+            reserve = max(self._GGUF_RESERVE_FLOOR_BYTES, int(total_b * self._GGUF_RESERVE_FRACTION))
+            need = int(weights_b) + int(kv_cache_bytes(geom, int(min_ctx), pad_v=True) or 0) + reserve
+        except Exception:
+            return None
+        from . import process_residency
+
+        def _free() -> int:
+            return int(torch.cuda.mem_get_info()[0])
+
+        return process_residency.ensure_fit(str(self.model), need, _free, provider="huggingface", device="GPU",
+                                            reserve_bytes=reserve)
+
     def _gguf_device_fit_rungs(self, model_path: Any, candidate_ctxs: List[int]) -> List[int]:
         """`candidate_ctxs` without the rungs whose KV cache does not fit the
         device under the GGUF context rule (above). Unchanged when the device
@@ -7378,6 +7433,15 @@ def _normalize_to_cache_format(self, model_name: str) -> str:
             return f"models--{model_name}"
 
     def _load_gguf_model(self):
+        """Load the GGUF under the process load lock (ADR 0010): the fit check and the
+        allocation it admits are one step, so two loads cannot both pass the check
+        against the same free memory and then run out of it."""
+        from .process_residency import load_lock
+
+        with load_lock():
+            return self._load_gguf_model_unlocked()
+
+    def _load_gguf_model_unlocked(self):
         """Load GGUF model using llama-cpp-python (cache-only, no downloading)"""
         import os
         try:
@@ -7509,6 +7573,14 @@ def _load_gguf_model(self):
             if requested_n_ctx_i <= 0:
                 requested_n_ctx_i = 16384
 
+            # THE FIT CHECK (core ADR 0010), before anything is allocated: the weights
+            # and the KV of the smallest context this load accepts must fit the free
+            # device memory, after evicting resident models that are not locked
+            # (least recently used first); else ModelDoesNotFitError, with numbers and
+            # smaller suggestions. Never a load that fills the card or the RAM.
+            _min_ctx = requested_n_ctx_i if getattr(self, "_user_provided_max_tokens", False) else min(4096, requested_n_ctx_i)
+            self._gguf_fit_report = self._gguf_ensure_fit(model_path, _min_ctx)
+
             if getattr(self, "_user_provided_max_tokens", False):
                 candidate_ctxs = [requested_n_ctx_i]
             else:
@@ -7583,6 +7655,11 @@ def _load_gguf_model(self):
                     is_metal_host = platform.system().lower() == "darwin" and platform.machine().lower() == "arm64"
                     cpu_retry_now = is_metal_host or n_ctx_i == candidate_ctxs[-1]
                     if isinstance(self.n_gpu_layers, int) and self.n_gpu_layers != 0 and cpu_retry_now:
+                        # ADR 0010: the CPU fallback puts weights AND the KV cache in RAM;
+                        # refuse it with the numbers when RAM cannot hold them, instead of
+                        # letting the kernel kill the process.
+                        if not is_metal_host:  # Metal retries every rung on the CPU: its ladder decides
+                            self._gguf_ensure_cpu_fit(model_path, int(n_ctx_i))
                         try:
                             self.logger.warning(
                                 f"GGUF load failed with n_gpu_layers={self.n_gpu_layers}; retrying with CPU (n_gpu_layers=0). Error: {e}"
@@ -7771,6 +7848,8 @@ def _load_gguf_model(self):
             except Exception:
                 pass
 
+        except ModelDoesNotFitError:
+            raise  # ADR 0010: the typed refusal reaches the caller as itself
         except ModelArtifactMismatchError:
             # ADR 0009 Enforcement: an artifact-fidelity refusal must reach the caller
             # as itself. Wrapping it in RuntimeError here made the contract type