From 8be3e8088e296089fb11a497d7ee9c0e7ad4988a Mon Sep 17 00:00:00 2001 From: iback Date: Fri, 13 Nov 2026 10:08:57 +0000 Subject: [PATCH] feat: mutually exclusive paths through the GPU pipeline with `pipeline.split` `"pipeline": {"split": {"paths": [[...], [...]], "p": [...]}}` draws one path per batch. The transforms named in the paths not taken are left out of that batch's forward sequence, so they neither draw parameters nor touch image or mask; the path taken runs as usual, each transform still subject to its own `p`; everything the split does not name runs on every batch. Pipeline order is unchanged, so a path need not be contiguous. The motivating case is SynthSeg on half the batches and the transfer augmentations on the other half, the general enhancements on all of them. Adding SynthSeg at p=0.5 to the paper config does not say that: it sits ahead of every TA, so on the batches it fires the TAs then run on the synthetic image. With paths [[SynthSeg], [the six TAs the SynthSeg paper config zeroes]] and SynthSeg at p=1, each path reproduces one of the two shipped paper configs exactly. Validated at load time: at least two paths, one `p` each, summing to 1, every name in the GPU section and in one path at most, sequential mode only (the bucketing modes hide the members inside a RandomChooseX), and not with `same_on_batch: false` (the draw is per batch). The container also refuses a name that is not one of its stages. The draw uses `shared_rand`, so it follows `torch.manual_seed` and DDP ranks agree on the path. The trainer builds its own container rather than an `AugTransformsGPU`, so it passes the split itself; a test constructs the trainer to pin that. Co-Authored-By: Claude Opus 5.5 --- smauglab/config.py | 93 +++++++++- smauglab/trainers/nnUNetTrainerDAExt.py | 4 + smauglab/transforms/gpu/base.py | 71 +++++++- smauglab/transforms/gpu/transforms.py | 1 + unit_tests/test_pipeline_split.py | 214 ++++++++++++++++++++++++ 5 files changed, 379 insertions(+), 4 deletions(-) create mode 100644 unit_tests/test_pipeline_split.py diff --git a/smauglab/config.py b/smauglab/config.py index 93bd7ca..c7a81a3 100644 --- a/smauglab/config.py +++ b/smauglab/config.py @@ -13,6 +13,9 @@ checked against the registry, and every problem in a file is reported at once rather than one per run. +`pipeline.split` (see `SplitSpec`) names mutually exclusive groups of GPU transforms; each +batch runs exactly one of them, alongside everything the split does not name. + A flat document with no section is rejected: it was interpreted as "GPU or CPU, whichever the keys look like", and the two namespaces overlapped enough that `GaussianBlurTransform` meant different transforms depending on which builder read it. `SECTION_HINT` below is what @@ -25,6 +28,8 @@ import difflib import functools import json +import math +from dataclasses import dataclass from enum import Enum from pathlib import Path from typing import Any @@ -96,7 +101,85 @@ def validate_section(section: dict, backend: Backend, *, source: str = " RESERVED_SECTIONS = ("pipeline",) #: Keys the `pipeline` section may hold. -PIPELINE_KEYS = ("mode", "order", "random_choose", "same_on_batch") +PIPELINE_KEYS = ("mode", "order", "random_choose", "same_on_batch", "split") + +#: Keys `pipeline.split` may hold. +SPLIT_KEYS = ("paths", "p") + + +@dataclass(frozen=True) +class SplitSpec: + """`pipeline.split`: mutually exclusive paths through the GPU pipeline. + + "pipeline": {"split": {"paths": [["RandomSynthSegGPU"], + ["RandomInverseGPU", "RandomScharrGPU"]], + "p": [0.5, 0.5]}} + + Each batch takes exactly one path, path `i` with probability `p[i]`. The transforms + named in the paths it did not take are skipped for that batch; those in the path it + took run as usual, each still subject to its own `p`; and every transform the split + does not name runs on every batch. Pipeline order is untouched -- a split decides + which transforms run, never in what order -- so a path need not be contiguous. + + The draw is one per batch, like every other probability under the default + `same_on_batch: true`, which is why the split refuses `same_on_batch: false`. An + empty path is allowed and means "none of the others". + """ + + paths: tuple[tuple[str, ...], ...] + p: tuple[float, ...] + + @classmethod + def from_payload(cls, raw: dict) -> SplitSpec: + return cls(tuple(tuple(path) for path in raw["paths"]), tuple(float(p) for p in raw["p"])) + + +def _split_problems(split: Any, pipeline: dict, gpu_section: Any) -> list[str]: + """Every problem with `pipeline.split`. Empty means `SplitSpec.from_payload` will build.""" + if not isinstance(split, dict): + return ["pipeline.split: expected an object with 'paths' and 'p'"] + problems = [ + f"pipeline.split: unknown key {key!r}. Accepted: {', '.join(SPLIT_KEYS)}." + for key in split + if not key.startswith("_") and key not in SPLIT_KEYS + ] + + paths, probabilities = split.get("paths"), split.get("p") + if not ( + isinstance(paths, list) + and len(paths) >= 2 + and all(isinstance(path, list) and all(isinstance(n, str) for n in path) for path in paths) + ): + problems.append("pipeline.split.paths: expected a list of at least two lists of GPU transform names") + paths = None + if not (isinstance(probabilities, list) and all(isinstance(p, (int, float)) and not isinstance(p, bool) for p in probabilities)): + problems.append("pipeline.split.p: expected a list of probabilities, one per path") + probabilities = None + elif any(not 0.0 <= p <= 1.0 for p in probabilities) or not math.isclose(sum(probabilities), 1.0, abs_tol=1e-6): + problems.append(f"pipeline.split.p: each must lie in [0, 1] and together they must sum to 1, got {probabilities}") + if paths is not None and probabilities is not None and len(paths) != len(probabilities): + problems.append(f"pipeline.split: {len(paths)} paths but {len(probabilities)} probabilities") + + if paths is not None: + configured = {k for k in gpu_section if not k.startswith("_")} if isinstance(gpu_section, dict) else set() + seen: set[str] = set() + for path in paths: + for name in path: + if name in seen: + problems.append(f"pipeline.split.paths: {name!r} is named twice; a transform belongs to one path at most") + seen.add(name) + if name not in configured: + close = difflib.get_close_matches(name, sorted(configured), n=2, cutoff=0.6) + hint = f" Did you mean: {', '.join(close)}?" if close else "" + problems.append(f"pipeline.split.paths: {name!r} is not in the GPU section.{hint}") + + mode = pipeline.get("mode") + if mode not in (None, PipelineMode.SEQUENTIAL.value): + problems.append(f"pipeline.split: needs the sequential pipeline; mode {mode!r} buckets the transforms the split would gate") + if pipeline.get("same_on_batch") is False: + problems.append("pipeline.split: draws one path per batch, so it cannot be combined with same_on_batch: false") + return problems + #: What to do about a config that has no backend section. Quoted by both errors that #: can report it, so the two cannot drift apart. @@ -176,6 +259,11 @@ def same_on_batch(self) -> bool: raw = self.payload.get("pipeline", {}).get("same_on_batch") return True if raw is None else bool(raw) + def pipeline_split(self) -> SplitSpec | None: + """The exclusive GPU paths `pipeline.split` asks for, or None without one.""" + raw = self.payload.get("pipeline", {}).get("split") + return None if raw is None else SplitSpec.from_payload(raw) + def names(self, backend: Backend) -> list[str]: return [k for k in self.section(backend) if not k.startswith("_")] @@ -246,6 +334,9 @@ def _pipeline_problems(self) -> list[str]: close = difflib.get_close_matches(str(mode), valid, n=2, cutoff=0.5) hint = f" Did you mean: {', '.join(close)}?" if close else "" problems.append(f"pipeline.mode: unknown mode {mode!r}. Accepted: {', '.join(valid)}.{hint}") + + if "split" in pipeline: + problems.extend(_split_problems(pipeline["split"], pipeline, self.payload.get(Backend.GPU.value))) return problems diff --git a/smauglab/trainers/nnUNetTrainerDAExt.py b/smauglab/trainers/nnUNetTrainerDAExt.py index a9dac85..dff6fc1 100644 --- a/smauglab/trainers/nnUNetTrainerDAExt.py +++ b/smauglab/trainers/nnUNetTrainerDAExt.py @@ -199,12 +199,16 @@ def __init__(self, plans: dict, configuration: str, fold: int, dataset_json: dic # See AugTransformsGPU: True forces the flag onto every child and # spends each transform's `p` once for the whole batch. same_on_batch=True if config.same_on_batch() else None, + split=config.pipeline_split(), ).to(self.device) print(f"Using SmaugLab transforms from: {json_path}") print( f" CPU: {len(config.names(Backend.CPU))} augmentations, GPU: {len(config.names(Backend.GPU))}, mode: {config.pipeline_mode().value}" ) + split = config.pipeline_split() + if split is not None: + print(f" split: one of {len(split.paths)} paths per batch, p={list(split.p)}, paths={[list(path) for path in split.paths]}") _record_config(json_path, os.path.join(self.output_folder, "transform_params_used_for_training.json")) diff --git a/smauglab/transforms/gpu/base.py b/smauglab/transforms/gpu/base.py index 65490ad..4523af6 100644 --- a/smauglab/transforms/gpu/base.py +++ b/smauglab/transforms/gpu/base.py @@ -2,8 +2,8 @@ import copy import functools import warnings -from collections.abc import Sequence -from typing import Any, Protocol, Union +from collections.abc import Iterator, Sequence +from typing import TYPE_CHECKING, Any, Protocol, Union import kornia.augmentation as K import torch @@ -32,6 +32,11 @@ from torch.distributions import RelaxedBernoulli from torch.nn import Module +from smauglab.transforms.rng import shared_rand + +if TYPE_CHECKING: + from smauglab.config import SplitSpec + try: # private to kornia, and the only part of the faster parameter draw that needs it from kornia.augmentation.container.image import ImageSequentialBase, _get_new_batch_shape @@ -138,7 +143,12 @@ def apply_transform_class( class AugmentationSequentialCustom(AugmentationSequential): - """Custom AugmentationSequential to handle masks augmentations.""" + """Custom AugmentationSequential to handle masks augmentations. + + `split` makes groups of stages mutually exclusive (see `smauglab.config.SplitSpec`): + one path is drawn per call, and the stages of every other path are left out of that + call's forward sequence, so they neither draw parameters nor touch image or mask. + """ def __init__( self, @@ -150,6 +160,7 @@ def __init__( random_apply_weights: list[float] | None = None, transformation_matrix_mode: str = "silent", extra_args: dict[DataKey, dict[str, Any]] | None = None, + split: "SplitSpec | None" = None, ) -> None: self._transform_matrix: Tensor | None self._transform_matrices: list[Tensor | None] = [] @@ -198,6 +209,60 @@ def __init__( self._transform_matrix = None self.extra_args = extra_args or {DataKey.MASK: {"resample": Resample.NEAREST, "align_corners": None}} + self.split_spec = split + self._split_paths = None if split is None else self._resolve_split(split) + + def _resolve_split(self, split: "SplitSpec") -> tuple[frozenset[str], ...]: + """Each path as the names of the child modules it covers. + + A config key is its transform's class name, so that is the match. Every name + must be exactly one stage here: a stage wrapped in a `RandomChooseX` bucket is + not a child, and gating something that is not there would make the split a + silent no-op. + """ + by_class: dict[str, list[str]] = {} + for child_name, child in self.named_children(): + by_class.setdefault(type(child).__name__, []).append(child_name) + paths = [] + for path in split.paths: + members = set() + for name in path: + found = by_class.get(name, []) + if len(found) != 1: + raise ValueError( + f"pipeline.split names {name!r}, which is {'not a stage' if not found else 'more than one stage'} of this pipeline" + ) + members.add(found[0]) + paths.append(frozenset(members)) + return tuple(paths) + + def _draw_skipped(self) -> frozenset[str]: + """Draw this call's path; return the stages of every other path.""" + assert self.split_spec is not None and self._split_paths is not None + # The shared draw, so DDP ranks take the same path for the same batch. + draw = float(shared_rand((1,), torch.device("cpu")).item()) + taken, cumulative = len(self.split_spec.p) - 1, 0.0 + for index, p in enumerate(self.split_spec.p): + cumulative += p + if draw < cumulative: + taken = index + break + return frozenset().union(*(path for index, path in enumerate(self._split_paths) if index != taken)) + + def get_forward_sequence(self, params: list[ParamItem] | None = None) -> Iterator[tuple[str, Module]]: + """kornia's forward sequence, minus the stages of the paths not taken. + + Only the parameter-drawing call (`params is None`) is filtered; `forward` then + iterates the parameters that call produced, so a skipped stage is skipped for + the image and the mask alike. Calls that replay `params` resolve stages from + them and are left alone. + """ + sequence = super().get_forward_sequence(params) + if params is not None or self._split_paths is None: + return sequence + skipped = self._draw_skipped() + return ((name, module) for name, module in sequence if name not in skipped) + def forward_parameters(self, batch_shape: torch.Size) -> list[ParamItem]: """kornia's own loop, with each leaf's draw routed through `_leaf_parameters`. diff --git a/smauglab/transforms/gpu/transforms.py b/smauglab/transforms/gpu/transforms.py index c57ed3c..90c2a59 100644 --- a/smauglab/transforms/gpu/transforms.py +++ b/smauglab/transforms/gpu/transforms.py @@ -36,4 +36,5 @@ def __init__(self, json_path: str): *transforms, data_keys=["input", "mask"], same_on_batch=True if config.same_on_batch() else None, + split=config.pipeline_split(), ) diff --git a/unit_tests/test_pipeline_split.py b/unit_tests/test_pipeline_split.py new file mode 100644 index 0000000..b505775 --- /dev/null +++ b/unit_tests/test_pipeline_split.py @@ -0,0 +1,214 @@ +"""`pipeline.split`: mutually exclusive paths through the GPU pipeline. + +The question it exists for: "SynthSeg on half the batches, our transfer augmentations +on the other half, the general enhancements on all of them". Stacking SynthSeg at +p=0.5 onto the paper pipeline is not that -- the transfer augmentations then also run on +the synthetic image -- and no other configuration surface could say "not both". + +What is pinned here: the config refuses every malformed split at load time; the paths +never co-occur; the shared stages run on every batch; the path frequencies follow `p`; +a skipped stage touches neither image nor mask; and the trainer, the one place a split +changes what a model is trained on, passes it through. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import tempfile +import unittest +from collections import Counter +from pathlib import Path +from typing import ClassVar +from unittest import mock + +import torch + +from smauglab.config import PipelineMode, SmaugConfig, SplitSpec +from smauglab.registry import Backend, InvalidConfigError +from smauglab.transforms.build import build_gpu_pipeline +from smauglab.transforms.gpu.base import AugmentationSequentialCustom, record_applications + +GPU = { + "RandomFlipTransformGPU": {"flip_axis": [0], "p": 1.0}, + "RandomInverseGPU": {"p": 1.0}, + "RandomGammaGPU": {"gamma_range": [0.7, 1.5], "p": 1.0}, + "RandomGaussianNoiseGPU": {"mean": 0.0, "std": 0.1, "p": 1.0}, +} +SHAPE = torch.Size((2, 1, 8, 8, 8)) + + +def config(split: dict | None, **pipeline) -> SmaugConfig: + payload: dict = {"GPU": json.loads(json.dumps(GPU))} + if split is not None: + pipeline["split"] = split + if pipeline: + payload["pipeline"] = pipeline + return SmaugConfig(payload) + + +def pipeline(cfg: SmaugConfig) -> AugmentationSequentialCustom: + return AugmentationSequentialCustom( + *build_gpu_pipeline(cfg.section(Backend.GPU), mode=cfg.pipeline_mode()), + data_keys=["input", "mask"], + same_on_batch=True, + split=cfg.pipeline_split(), + ) + + +def stages(pipe: AugmentationSequentialCustom) -> set[str]: + """The class names of the stages one parameter draw selected.""" + return {param.name.rsplit("_", 1)[0] for param in pipe.forward_parameters(SHAPE)} + + +class TestSplitConfig(unittest.TestCase): + SPLIT: ClassVar[dict] = {"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]} + + def assertRejected(self, split: dict | object, fragment: str, **pipeline) -> None: + with self.assertRaises(InvalidConfigError) as caught: + config(split, **pipeline) # type: ignore[arg-type] + self.assertIn(fragment, str(caught.exception)) + + def test_absent_means_no_split(self): + self.assertIsNone(config(None).pipeline_split()) + + def test_a_valid_split_parses(self): + self.assertEqual( + config(self.SPLIT).pipeline_split(), + SplitSpec(paths=(("RandomInverseGPU",), ("RandomGammaGPU",)), p=(0.5, 0.5)), + ) + + def test_an_empty_path_is_allowed(self): + spec = config({"paths": [["RandomInverseGPU"], []], "p": [0.3, 0.7]}).pipeline_split() + self.assertEqual(spec, SplitSpec(paths=(("RandomInverseGPU",), ()), p=(0.3, 0.7))) + + def test_a_name_missing_from_the_gpu_section_is_rejected_with_a_hint(self): + self.assertRejected({"paths": [["RandomInversGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]}, "Did you mean: RandomInverseGPU") + + def test_a_name_in_two_paths_is_rejected(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomInverseGPU"]], "p": [0.5, 0.5]}, "named twice") + + def test_probabilities_must_sum_to_one(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.4]}, "sum to 1") + + def test_probabilities_must_lie_in_the_unit_interval(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [1.5, -0.5]}, "[0, 1]") + + def test_one_probability_per_path(self): + self.assertRejected({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [1.0]}, "2 paths but 1 probabilities") + + def test_a_single_path_is_not_a_split(self): + self.assertRejected({"paths": [["RandomInverseGPU"]], "p": [1.0]}, "at least two") + + def test_an_unknown_key_is_rejected(self): + self.assertRejected({**self.SPLIT, "weights": [1, 1]}, "unknown key 'weights'") + + def test_a_non_object_is_rejected(self): + self.assertRejected(["RandomInverseGPU"], "expected an object") + + def test_a_bucketing_mode_is_rejected(self): + self.assertRejected(self.SPLIT, "needs the sequential pipeline", mode=PipelineMode.RANDOM_ORDER.value) + + def test_per_sample_draws_are_rejected(self): + self.assertRejected(self.SPLIT, "same_on_batch: false", same_on_batch=False) + + +class TestSplitContainer(unittest.TestCase): + def test_no_split_keeps_every_stage(self): + pipe = pipeline(config(None)) + self.assertEqual(stages(pipe), set(GPU)) + + def test_paths_never_co_occur_and_shared_stages_always_run(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.25, 0.75]})) + torch.manual_seed(0) + n, taken = 4000, Counter() + for _ in range(n): + drawn = stages(pipe) + self.assertTrue({"RandomFlipTransformGPU", "RandomGaussianNoiseGPU"} <= drawn, drawn) + self.assertEqual(len(drawn & {"RandomInverseGPU", "RandomGammaGPU"}), 1, drawn) + taken["inverse" if "RandomInverseGPU" in drawn else "gamma"] += 1 + # Binomial(4000, 0.25): sd 27.4, so +-150 is over five sd. + self.assertLess(abs(taken["inverse"] - 0.25 * n), 150, taken) + + def test_an_empty_path_runs_neither_of_the_others(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU", "RandomGammaGPU"], []], "p": [0.5, 0.5]})) + torch.manual_seed(1) + seen = Counter(frozenset(stages(pipe) & {"RandomInverseGPU", "RandomGammaGPU"}) for _ in range(400)) + self.assertEqual(set(seen), {frozenset({"RandomInverseGPU", "RandomGammaGPU"}), frozenset()}, seen) + + def test_the_draw_follows_the_torch_seed(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]})) + runs = [] + for _ in range(2): + torch.manual_seed(7) + runs.append([frozenset(stages(pipe)) for _ in range(50)]) + self.assertEqual(runs[0], runs[1]) + + def test_a_skipped_stage_touches_neither_image_nor_mask(self): + pipe = pipeline(config({"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]})) + torch.manual_seed(2) + x = torch.rand(SHAPE) + 0.1 + seg = (torch.rand(SHAPE) > 0.5).float() + taken = Counter() + for _ in range(40): + with record_applications(pipe) as fired: + _, mask = pipe(x.clone(), seg.clone()) + ran = {entry["transform"] for entry in fired if entry["target"] == "image"} + self.assertEqual(len(ran & {"RandomInverseGPU", "RandomGammaGPU"}), 1, ran) + self.assertIn("RandomGaussianNoiseGPU", ran) + taken[frozenset(ran)] += 1 + # Flip at p=1 is the only stage that moves the mask, whichever path ran. + self.assertTrue(torch.equal(mask, seg.flip(2))) + self.assertEqual(len(taken), 2, taken) + + def test_a_name_that_is_not_a_stage_raises(self): + # In a bucketing mode the members live inside a RandomChooseX and are not stages; + # gating them would silently do nothing. The config refuses this combination, so + # this is the direct-construction guard. + cfg = config(None) + built = build_gpu_pipeline(cfg.section(Backend.GPU), mode=PipelineMode.RANDOM_ORDER_TA) + spec = SplitSpec(paths=(("RandomInverseGPU",), ("RandomGammaGPU",)), p=(0.5, 0.5)) + with self.assertRaisesRegex(ValueError, "not a stage"): + AugmentationSequentialCustom(*built, data_keys=["input", "mask"], same_on_batch=True, split=spec) + + def test_aug_transforms_gpu_reads_the_split(self): + from smauglab.transforms.gpu.transforms import AugTransformsGPU + + payload = {"GPU": GPU, "pipeline": {"split": {"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]}}} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "transform_params_split_aug.json" + path.write_text(json.dumps(payload)) + pipe = AugTransformsGPU(str(path)) + self.assertEqual(pipe.split_spec, SmaugConfig(payload).pipeline_split()) + + +@unittest.skipIf(importlib.util.find_spec("nnunetv2") is None, "the trainer needs the nnunetv2 extra") +class TestTrainerPassesTheSplit(unittest.TestCase): + """The trainer builds its own container rather than an `AugTransformsGPU`, so it has to + pass the split itself -- `pipeline.order` was once dropped at exactly this spot.""" + + def test_the_trainer_pipeline_carries_the_split(self): + from nnunetv2.training.nnUNetTrainer.nnUNetTrainer import nnUNetTrainer + + from smauglab.trainers.nnUNetTrainerDAExt import nnUNetTrainerDAExtGPU + + payload = {"GPU": GPU, "pipeline": {"split": {"paths": [["RandomInverseGPU"], ["RandomGammaGPU"]], "p": [0.5, 0.5]}}} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "transform_params_split_trainer.json" + path.write_text(json.dumps(payload)) + + def bare_init(self, *_args, **_kwargs): + self.device = torch.device("cpu") + self.output_folder = tmp + + with mock.patch.object(nnUNetTrainer, "__init__", bare_init), mock.patch.dict(os.environ, {"SMAUGLAB_PARAMS_JSON": str(path)}): + trainer = nnUNetTrainerDAExtGPU({}, "3d_fullres", 0, {}) + + assert trainer.transforms is not None + self.assertEqual(trainer.transforms.split_spec, SmaugConfig(payload).pipeline_split()) + self.assertEqual(len(trainer.transforms._split_paths or ()), 2) + + +if __name__ == "__main__": + unittest.main()