diff --git a/smauglab/config.py b/smauglab/config.py index 93bd7ca..c7a81a3 100644 --- a/smauglab/config.py +++ b/smauglab/config.py @@ -13,6 +13,9 @@ checked against the registry, and every problem in a file is reported at once rather than one per run. +`pipeline.split` (see `SplitSpec`) names mutually exclusive groups of GPU transforms; each +batch runs exactly one of them, alongside everything the split does not name. + A flat document with no section is rejected: it was interpreted as "GPU or CPU, whichever the keys look like", and the two namespaces overlapped enough that `GaussianBlurTransform` meant different transforms depending on which builder read it. `SECTION_HINT` below is what @@ -25,6 +28,8 @@ import difflib import functools import json +import math +from dataclasses import dataclass from enum import Enum from pathlib import Path from typing import Any @@ -96,7 +101,85 @@ def validate_section(section: dict, backend: Backend, *, source: str = " RESERVED_SECTIONS = ("pipeline",) #: Keys the `pipeline` section may hold. -PIPELINE_KEYS = ("mode", "order", "random_choose", "same_on_batch") +PIPELINE_KEYS = ("mode", "order", "random_choose", "same_on_batch", "split") + +#: Keys `pipeline.split` may hold. +SPLIT_KEYS = ("paths", "p") + + +@dataclass(frozen=True) +class SplitSpec: + """`pipeline.split`: mutually exclusive paths through the GPU pipeline. + + "pipeline": {"split": {"paths": [["RandomSynthSegGPU"], + ["RandomInverseGPU", "RandomScharrGPU"]], + "p": [0.5, 0.5]}} + + Each batch takes exactly one path, path `i` with probability `p[i]`. The transforms + named in the paths it did not take are skipped for that batch; those in the path it + took run as usual, each still subject to its own `p`; and every transform the split + does not name runs on every batch. Pipeline order is untouched -- a split decides + which transforms run, never in what order -- so a path need not be contiguous. + + The draw is one per batch, like every other probability under the default + `same_on_batch: true`, which is why the split refuses `same_on_batch: false`. An + empty path is allowed and means "none of the others". + """ + + paths: tuple[tuple[str, ...], ...] + p: tuple[float, ...] + + @classmethod + def from_payload(cls, raw: dict) -> SplitSpec: + return cls(tuple(tuple(path) for path in raw["paths"]), tuple(float(p) for p in raw["p"])) + + +def _split_problems(split: Any, pipeline: dict, gpu_section: Any) -> list[str]: + """Every problem with `pipeline.split`. Empty means `SplitSpec.from_payload` will build.""" + if not isinstance(split, dict): + return ["pipeline.split: expected an object with 'paths' and 'p'"] + problems = [ + f"pipeline.split: unknown key {key!r}. Accepted: {', '.join(SPLIT_KEYS)}." + for key in split + if not key.startswith("_") and key not in SPLIT_KEYS + ] + + paths, probabilities = split.get("paths"), split.get("p") + if not ( + isinstance(paths, list) + and len(paths) >= 2 + and all(isinstance(path, list) and all(isinstance(n, str) for n in path) for path in paths) + ): + problems.append("pipeline.split.paths: expected a list of at least two lists of GPU transform names") + paths = None + if not (isinstance(probabilities, list) and all(isinstance(p, (int, float)) and not isinstance(p, bool) for p in probabilities)): + problems.append("pipeline.split.p: expected a list of probabilities, one per path") + probabilities = None + elif any(not 0.0 <= p <= 1.0 for p in probabilities) or not math.isclose(sum(probabilities), 1.0, abs_tol=1e-6): + problems.append(f"pipeline.split.p: each must lie in [0, 1] and together they must sum to 1, got {probabilities}") + if paths is not None and probabilities is not None and len(paths) != len(probabilities): + problems.append(f"pipeline.split: {len(paths)} paths but {len(probabilities)} probabilities") + + if paths is not None: + configured = {k for k in gpu_section if not k.startswith("_")} if isinstance(gpu_section, dict) else set() + seen: set[str] = set() + for path in paths: + for name in path: + if name in seen: + problems.append(f"pipeline.split.paths: {name!r} is named twice; a transform belongs to one path at most") + seen.add(name) + if name not in configured: + close = difflib.get_close_matches(name, sorted(configured), n=2, cutoff=0.6) + hint = f" Did you mean: {', '.join(close)}?" if close else "" + problems.append(f"pipeline.split.paths: {name!r} is not in the GPU section.{hint}") + + mode = pipeline.get("mode") + if mode not in (None, PipelineMode.SEQUENTIAL.value): + problems.append(f"pipeline.split: needs the sequential pipeline; mode {mode!r} buckets the transforms the split would gate") + if pipeline.get("same_on_batch") is False: + problems.append("pipeline.split: draws one path per batch, so it cannot be combined with same_on_batch: false") + return problems + #: What to do about a config that has no backend section. Quoted by both errors that #: can report it, so the two cannot drift apart. @@ -176,6 +259,11 @@ def same_on_batch(self) -> bool: raw = self.payload.get("pipeline", {}).get("same_on_batch") return True if raw is None else bool(raw) + def pipeline_split(self) -> SplitSpec | None: + """The exclusive GPU paths `pipeline.split` asks for, or None without one.""" + raw = self.payload.get("pipeline", {}).get("split") + return None if raw is None else SplitSpec.from_payload(raw) + def names(self, backend: Backend) -> list[str]: return [k for k in self.section(backend) if not k.startswith("_")] @@ -246,6 +334,9 @@ def _pipeline_problems(self) -> list[str]: close = difflib.get_close_matches(str(mode), valid, n=2, cutoff=0.5) hint = f" Did you mean: {', '.join(close)}?" if close else "" problems.append(f"pipeline.mode: unknown mode {mode!r}. Accepted: {', '.join(valid)}.{hint}") + + if "split" in pipeline: + problems.extend(_split_problems(pipeline["split"], pipeline, self.payload.get(Backend.GPU.value))) return problems diff --git a/smauglab/trainers/nnUNetTrainerDAExt.py b/smauglab/trainers/nnUNetTrainerDAExt.py index a9dac85..dff6fc1 100644 --- a/smauglab/trainers/nnUNetTrainerDAExt.py +++ b/smauglab/trainers/nnUNetTrainerDAExt.py @@ -199,12 +199,16 @@ def __init__(self, plans: dict, configuration: str, fold: int, dataset_json: dic # See AugTransformsGPU: True forces the flag onto every child and # spends each transform's `p` once for the whole batch. same_on_batch=True if config.same_on_batch() else None, + split=config.pipeline_split(), ).to(self.device) print(f"Using SmaugLab transforms from: {json_path}") print( f" CPU: {len(config.names(Backend.CPU))} augmentations, GPU: {len(config.names(Backend.GPU))}, mode: {config.pipeline_mode().value}" ) + split = config.pipeline_split() + if split is not None: + print(f" split: one of {len(split.paths)} paths per batch, p={list(split.p)}, paths={[list(path) for path in split.paths]}") _record_config(json_path, os.path.join(self.output_folder, "transform_params_used_for_training.json")) diff --git a/smauglab/transforms/gpu/base.py b/smauglab/transforms/gpu/base.py index 65490ad..4523af6 100644 --- a/smauglab/transforms/gpu/base.py +++ b/smauglab/transforms/gpu/base.py @@ -2,8 +2,8 @@ import copy import functools import warnings -from collections.abc import Sequence -from typing import Any, Protocol, Union +from collections.abc import Iterator, Sequence +from typing import TYPE_CHECKING, Any, Protocol, Union import kornia.augmentation as K import torch @@ -32,6 +32,11 @@ from torch.distributions import RelaxedBernoulli from torch.nn import Module +from smauglab.transforms.rng import shared_rand + +if TYPE_CHECKING: + from smauglab.config import SplitSpec + try: # private to kornia, and the only part of the faster parameter draw that needs it from kornia.augmentation.container.image import ImageSequentialBase, _get_new_batch_shape @@ -138,7 +143,12 @@ def apply_transform_class( class AugmentationSequentialCustom(AugmentationSequential): - """Custom AugmentationSequential to handle masks augmentations.""" + """Custom AugmentationSequential to handle masks augmentations. + + `split` makes groups of stages mutually exclusive (see `smauglab.config.SplitSpec`): + one path is drawn per call, and the stages of every other path are left out of that + call's forward sequence, so they neither draw parameters nor touch image or mask. + """ def __init__( self, @@ -150,6 +160,7 @@ def __init__( random_apply_weights: list[float] | None = None, transformation_matrix_mode: str = "silent", extra_args: dict[DataKey, dict[str, Any]] | None = None, + split: "SplitSpec | None" = None, ) -> None: self._transform_matrix: Tensor | None self._transform_matrices: list[Tensor | None] = [] @@ -198,6 +209,60 @@ def __init__( self._transform_matrix = None self.extra_args = extra_args or {DataKey.MASK: {"resample": Resample.NEAREST, "align_corners": None}} + self.split_spec = split + self._split_paths = None if split is None else self._resolve_split(split) + + def _resolve_split(self, split: "SplitSpec") -> tuple[frozenset[str], ...]: + """Each path as the names of the child modules it covers. + + A config key is its transform's class name, so that is the match. Every name + must be exactly one stage here: a stage wrapped in a `RandomChooseX` bucket is + not a child, and gating something that is not there would make the split a + silent no-op. + """ + by_class: dict[str, list[str]] = {} + for child_name, child in self.named_children(): + by_class.setdefault(type(child).__name__, []).append(child_name) + paths = [] + for path in split.paths: + members = set() + for name in path: + found = by_class.get(name, []) + if len(found) != 1: + raise ValueError( + f"pipeline.split names {name!r}, which is {'not a stage' if not found else 'more than one stage'} of this pipeline" + ) + members.add(found[0]) + paths.append(frozenset(members)) + return tuple(paths) + + def _draw_skipped(self) -> frozenset[str]: + """Draw this call's path; return the stages of every other path.""" + assert self.split_spec is not None and self._split_paths is not None + # The shared draw, so DDP ranks take the same path for the same batch. + draw = float(shared_rand((1,), torch.device("cpu")).item()) + taken, cumulative = len(self.split_spec.p) - 1, 0.0 + for index, p in enumerate(self.split_spec.p): + cumulative += p + if draw < cumulative: + taken = index + break + return frozenset().union(*(path for index, path in enumerate(self._split_paths) if index != taken)) + + def get_forward_sequence(self, params: list[ParamItem] | None = None) -> Iterator[tuple[str, Module]]: + """kornia's forward sequence, minus the stages of the paths not taken. + + Only the parameter-drawing call (`params is None`) is filtered; `forward` then + iterates the parameters that call produced, so a skipped stage is skipped for + the image and the mask alike. Calls that replay `params` resolve stages from + them and are left alone. + """ + sequence = super().get_forward_sequence(params) + if params is not None or self._split_paths is None: + return sequence + skipped = self._draw_skipped() + return ((name, module) for name, module in sequence if name not in skipped) + def forward_parameters(self, batch_shape: torch.Size) -> list[ParamItem]: """kornia's own loop, with each leaf's draw routed through `_leaf_parameters`. diff --git a/smauglab/transforms/gpu/transforms.py b/smauglab/transforms/gpu/transforms.py index c57ed3c..90c2a59 100644 --- a/smauglab/transforms/gpu/transforms.py +++ b/smauglab/transforms/gpu/transforms.py @@ -36,4 +36,5 @@ def __init__(self, json_path: str): *transforms, data_keys=["input", "mask"], same_on_batch=True if config.same_on_batch() else None, + split=config.pipeline_split(), ) diff --git a/unit_tests/test_pipeline_split.py b/unit_tests/test_pipeline_split.py new file mode 100644 index 0000000..b505775 --- /dev/null +++ b/unit_tests/test_pipeline_split.py @@ -0,0 +1,214 @@ +"""`pipeline.split`: mutually exclusive paths through the GPU pipeline. + +The question it exists for: "SynthSeg on half the batches, our transfer augmentations +on the other half, the general enhancements on all of them". Stacking SynthSeg at +p=0.5 onto the paper pipeline is not that -- the transfer augmentations then also run on +the synthetic image -- and no other configuration surface could say "not both". + +What is pinned here: the config refuses every malformed split at load time; the paths +never co-occur; the shared stages run on every batch; the path frequencies follow `p`; +a skipped stage touches neither image nor mask; and the trainer, the one place a split +changes what a model is trained on, passes it through. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import tempfile +import unittest +from collections import Counter +from pathlib import Path +from typing import ClassVar +from unittest import mock + +import torch + +from smauglab.config import PipelineMode, SmaugConfig, SplitSpec +from smauglab.registry import Backend, InvalidConfigError +from smauglab.transforms.build import build_gpu_pipeline +from smauglab.transforms.gpu.base import AugmentationSequentialCustom, record_applications + +GPU = { + "RandomFlipTransformGPU": {"flip_axis": [0], "p": 1.0}, + "RandomInverseGPU": {"p": 1.0}, + "RandomGammaGPU": {"gamma_range": [0.7, 1.5], "p": 1.0}, + "RandomGaussianNoiseGPU": {"mean": 0.0, "std": 0.1, "p": 1.0}, +} +SHAPE = torch.Size((2, 1, 8, 8, 8)) + + +def config(split: dict | None, **pipeline) -> SmaugConfig: + payload: dict = {"GPU": json.loads(json.dumps(GPU))} + if split is not None: + pipeline["split"] = split + if pipeline: + payload["pipeline"] = pipeline + return SmaugConfig(payload) + + +def pipeline(cfg: SmaugConfig) -> AugmentationSequentialCustom: + return AugmentationSequentialCustom( + *build_gpu_pipeline(cfg.section(Backend.GPU), mode=cfg.pipeline_mode()), + data_keys=["input", "mask"], + same_on_batch=True, + split=cfg.pipeline_split(), + ) + + +def stages(pipe: AugmentationSequentialCustom) -> set[str]: + """The class names of the stages one parameter draw selected.""" + return {param.name.rsplit("_", 1)[0] for param in pipe.forward_parameters(SHAPE)} + + +class TestSplitConfig(unittest.TestCase): + SPLIT: ClassVar[dict] = {"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]} + + def assertRejected(self, split: dict | object, fragment: str, **pipeline) -> None: + with self.assertRaises(InvalidConfigError) as caught: + config(split, **pipeline) # type: ignore[arg-type] + self.assertIn(fragment, str(caught.exception)) + + def test_absent_means_no_split(self): + self.assertIsNone(config(None).pipeline_split()) + + def test_a_valid_split_parses(self): + self.assertEqual( + config(self.SPLIT).pipeline_split(), + SplitSpec(paths=(("RandomInverseGPU",), ("RandomGammaGPU",)), p=(0.5, 0.5)), + ) + + def test_an_empty_path_is_allowed(self): + spec = config({"paths": [["RandomInverseGPU"], []], "p": [0.3, 0.7]}).pipeline_split() + self.assertEqual(spec, SplitSpec(paths=(("RandomInverseGPU",), ()), p=(0.3, 0.7))) + + def test_a_name_missing_from_the_gpu_section_is_rejected_with_a_hint(self): + self.assertRejected({"paths": [["RandomInversGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]}, "Did you mean: RandomInverseGPU") + + def test_a_name_in_two_paths_is_rejected(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomInverseGPU"]], "p": [0.5, 0.5]}, "named twice") + + def test_probabilities_must_sum_to_one(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.4]}, "sum to 1") + + def test_probabilities_must_lie_in_the_unit_interval(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [1.5, -0.5]}, "[0, 1]") + + def test_one_probability_per_path(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [1.0]}, "2 paths but 1 probabilities") + + def test_a_single_path_is_not_a_split(self): + self.assertRejected({"paths": [["RandomInverseGPU"]], "p": [1.0]}, "at least two") + + def test_an_unknown_key_is_rejected(self): + self.assertRejected({**self.SPLIT, "weights": [1, 1]}, "unknown key 'weights'") + + def test_a_non_object_is_rejected(self): + self.assertRejected(["RandomInverseGPU"], "expected an object") + + def test_a_bucketing_mode_is_rejected(self): + self.assertRejected(self.SPLIT, "needs the sequential pipeline", mode=PipelineMode.RANDOM_ORDER.value) + + def test_per_sample_draws_are_rejected(self): + self.assertRejected(self.SPLIT, "same_on_batch: false", same_on_batch=False) + + +class TestSplitContainer(unittest.TestCase): + def test_no_split_keeps_every_stage(self): + pipe = pipeline(config(None)) + self.assertEqual(stages(pipe), set(GPU)) + + def test_paths_never_co_occur_and_shared_stages_always_run(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.25, 0.75]})) + torch.manual_seed(0) + n, taken = 4000, Counter() + for _ in range(n): + drawn = stages(pipe) + self.assertTrue({"RandomFlipTransformGPU", "RandomGaussianNoiseGPU"} <= drawn, drawn) + self.assertEqual(len(drawn & {"RandomInverseGPU", "RandomGammaGPU"}), 1, drawn) + taken["inverse" if "RandomInverseGPU" in drawn else "gamma"] += 1 + # Binomial(4000, 0.25): sd 27.4, so +-150 is over five sd. + self.assertLess(abs(taken["inverse"] - 0.25 * n), 150, taken) + + def test_an_empty_path_runs_neither_of_the_others(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU", "RandomGammaGPU"], []], "p": [0.5, 0.5]})) + torch.manual_seed(1) + seen = Counter(frozenset(stages(pipe) & {"RandomInverseGPU", "RandomGammaGPU"}) for _ in range(400)) + self.assertEqual(set(seen), {frozenset({"RandomInverseGPU", "RandomGammaGPU"}), frozenset()}, seen) + + def test_the_draw_follows_the_torch_seed(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]})) + runs = [] + for _ in range(2): + torch.manual_seed(7) + runs.append([frozenset(stages(pipe)) for _ in range(50)]) + self.assertEqual(runs[0], runs[1]) + + def test_a_skipped_stage_touches_neither_image_nor_mask(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]})) + torch.manual_seed(2) + x = torch.rand(SHAPE) + 0.1 + seg = (torch.rand(SHAPE) > 0.5).float() + taken = Counter() + for _ in range(40): + with record_applications(pipe) as fired: + _, mask = pipe(x.clone(), seg.clone()) + ran = {entry["transform"] for entry in fired if entry["target"] == "image"} + self.assertEqual(len(ran & {"RandomInverseGPU", "RandomGammaGPU"}), 1, ran) + self.assertIn("RandomGaussianNoiseGPU", ran) + taken[frozenset(ran)] += 1 + # Flip at p=1 is the only stage that moves the mask, whichever path ran. + self.assertTrue(torch.equal(mask, seg.flip(2))) + self.assertEqual(len(taken), 2, taken) + + def test_a_name_that_is_not_a_stage_raises(self): + # In a bucketing mode the members live inside a RandomChooseX and are not stages; + # gating them would silently do nothing. The config refuses this combination, so + # this is the direct-construction guard. + cfg = config(None) + built = build_gpu_pipeline(cfg.section(Backend.GPU), mode=PipelineMode.RANDOM_ORDER_TA) + spec = SplitSpec(paths=(("RandomInverseGPU",), ("RandomGammaGPU",)), p=(0.5, 0.5)) + with self.assertRaisesRegex(ValueError, "not a stage"): + AugmentationSequentialCustom(*built, data_keys=["input", "mask"], same_on_batch=True, split=spec) + + def test_aug_transforms_gpu_reads_the_split(self): + from smauglab.transforms.gpu.transforms import AugTransformsGPU + + payload = {"GPU": GPU, "pipeline": {"split": {"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]}}} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "transform_params_split_aug.json" + path.write_text(json.dumps(payload)) + pipe = AugTransformsGPU(str(path)) + self.assertEqual(pipe.split_spec, SmaugConfig(payload).pipeline_split()) + + +@unittest.skipIf(importlib.util.find_spec("nnunetv2") is None, "the trainer needs the nnunetv2 extra") +class TestTrainerPassesTheSplit(unittest.TestCase): + """The trainer builds its own container rather than an `AugTransformsGPU`, so it has to + pass the split itself -- `pipeline.order` was once dropped at exactly this spot.""" + + def test_the_trainer_pipeline_carries_the_split(self): + from nnunetv2.training.nnUNetTrainer.nnUNetTrainer import nnUNetTrainer + + from smauglab.trainers.nnUNetTrainerDAExt import nnUNetTrainerDAExtGPU + + payload = {"GPU": GPU, "pipeline": {"split": {"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]}}} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "transform_params_split_trainer.json" + path.write_text(json.dumps(payload)) + + def bare_init(self, *_args, **_kwargs): + self.device = torch.device("cpu") + self.output_folder = tmp + + with mock.patch.object(nnUNetTrainer, "__init__", bare_init), mock.patch.dict(os.environ, {"SMAUGLAB_PARAMS_JSON": str(path)}): + trainer = nnUNetTrainerDAExtGPU({}, "3d_fullres", 0, {}) + + assert trainer.transforms is not None + self.assertEqual(trainer.transforms.split_spec, SmaugConfig(payload).pipeline_split()) + self.assertEqual(len(trainer.transforms._split_paths or ()), 2) + + +if __name__ == "__main__": + unittest.main()