Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 92 additions & 1 deletion smauglab/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
checked against the registry, and every problem in a file is reported at once rather than
one per run.

`pipeline.split` (see `SplitSpec`) names mutually exclusive groups of GPU transforms; each
batch runs exactly one of them, alongside everything the split does not name.

A flat document with no section is rejected: it was interpreted as "GPU or CPU, whichever
the keys look like", and the two namespaces overlapped enough that `GaussianBlurTransform`
meant different transforms depending on which builder read it. `SECTION_HINT` below is what
Expand All @@ -25,6 +28,8 @@
import difflib
import functools
import json
import math
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -96,7 +101,85 @@ def validate_section(section: dict, backend: Backend, *, source: str = "<config>
RESERVED_SECTIONS = ("pipeline",)

#: Keys the `pipeline` section may hold.
PIPELINE_KEYS = ("mode", "order", "random_choose", "same_on_batch")
PIPELINE_KEYS = ("mode", "order", "random_choose", "same_on_batch", "split")

#: Keys `pipeline.split` may hold.
SPLIT_KEYS = ("paths", "p")


@dataclass(frozen=True)
class SplitSpec:
"""`pipeline.split`: mutually exclusive paths through the GPU pipeline.

"pipeline": {"split": {"paths": [["RandomSynthSegGPU"],
["RandomInverseGPU", "RandomScharrGPU"]],
"p": [0.5, 0.5]}}

Each batch takes exactly one path, path `i` with probability `p[i]`. The transforms
named in the paths it did not take are skipped for that batch; those in the path it
took run as usual, each still subject to its own `p`; and every transform the split
does not name runs on every batch. Pipeline order is untouched -- a split decides
which transforms run, never in what order -- so a path need not be contiguous.

The draw is one per batch, like every other probability under the default
`same_on_batch: true`, which is why the split refuses `same_on_batch: false`. An
empty path is allowed and means "none of the others".
"""

paths: tuple[tuple[str, ...], ...]
p: tuple[float, ...]

@classmethod
def from_payload(cls, raw: dict) -> SplitSpec:
return cls(tuple(tuple(path) for path in raw["paths"]), tuple(float(p) for p in raw["p"]))


def _split_problems(split: Any, pipeline: dict, gpu_section: Any) -> list[str]:
"""Every problem with `pipeline.split`. Empty means `SplitSpec.from_payload` will build."""
if not isinstance(split, dict):
return ["pipeline.split: expected an object with 'paths' and 'p'"]
problems = [
f"pipeline.split: unknown key {key!r}. Accepted: {', '.join(SPLIT_KEYS)}."
for key in split
if not key.startswith("_") and key not in SPLIT_KEYS
]

paths, probabilities = split.get("paths"), split.get("p")
if not (
isinstance(paths, list)
and len(paths) >= 2
and all(isinstance(path, list) and all(isinstance(n, str) for n in path) for path in paths)
):
problems.append("pipeline.split.paths: expected a list of at least two lists of GPU transform names")
paths = None
if not (isinstance(probabilities, list) and all(isinstance(p, (int, float)) and not isinstance(p, bool) for p in probabilities)):
problems.append("pipeline.split.p: expected a list of probabilities, one per path")
probabilities = None
elif any(not 0.0 <= p <= 1.0 for p in probabilities) or not math.isclose(sum(probabilities), 1.0, abs_tol=1e-6):
problems.append(f"pipeline.split.p: each must lie in [0, 1] and together they must sum to 1, got {probabilities}")
if paths is not None and probabilities is not None and len(paths) != len(probabilities):
problems.append(f"pipeline.split: {len(paths)} paths but {len(probabilities)} probabilities")

if paths is not None:
configured = {k for k in gpu_section if not k.startswith("_")} if isinstance(gpu_section, dict) else set()
seen: set[str] = set()
for path in paths:
for name in path:
if name in seen:
problems.append(f"pipeline.split.paths: {name!r} is named twice; a transform belongs to one path at most")
seen.add(name)
if name not in configured:
close = difflib.get_close_matches(name, sorted(configured), n=2, cutoff=0.6)
hint = f" Did you mean: {', '.join(close)}?" if close else ""
problems.append(f"pipeline.split.paths: {name!r} is not in the GPU section.{hint}")

mode = pipeline.get("mode")
if mode not in (None, PipelineMode.SEQUENTIAL.value):
problems.append(f"pipeline.split: needs the sequential pipeline; mode {mode!r} buckets the transforms the split would gate")
if pipeline.get("same_on_batch") is False:
problems.append("pipeline.split: draws one path per batch, so it cannot be combined with same_on_batch: false")
return problems


#: What to do about a config that has no backend section. Quoted by both errors that
#: can report it, so the two cannot drift apart.
Expand Down Expand Up @@ -176,6 +259,11 @@ def same_on_batch(self) -> bool:
raw = self.payload.get("pipeline", {}).get("same_on_batch")
return True if raw is None else bool(raw)

def pipeline_split(self) -> SplitSpec | None:
"""The exclusive GPU paths `pipeline.split` asks for, or None without one."""
raw = self.payload.get("pipeline", {}).get("split")
return None if raw is None else SplitSpec.from_payload(raw)

def names(self, backend: Backend) -> list[str]:
return [k for k in self.section(backend) if not k.startswith("_")]

Expand Down Expand Up @@ -246,6 +334,9 @@ def _pipeline_problems(self) -> list[str]:
close = difflib.get_close_matches(str(mode), valid, n=2, cutoff=0.5)
hint = f" Did you mean: {', '.join(close)}?" if close else ""
problems.append(f"pipeline.mode: unknown mode {mode!r}. Accepted: {', '.join(valid)}.{hint}")

if "split" in pipeline:
problems.extend(_split_problems(pipeline["split"], pipeline, self.payload.get(Backend.GPU.value)))
return problems


Expand Down
4 changes: 4 additions & 0 deletions smauglab/trainers/nnUNetTrainerDAExt.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,12 +199,16 @@ def __init__(self, plans: dict, configuration: str, fold: int, dataset_json: dic
# See AugTransformsGPU: True forces the flag onto every child and
# spends each transform's `p` once for the whole batch.
same_on_batch=True if config.same_on_batch() else None,
split=config.pipeline_split(),
).to(self.device)

print(f"Using SmaugLab transforms from: {json_path}")
print(
f" CPU: {len(config.names(Backend.CPU))} augmentations, GPU: {len(config.names(Backend.GPU))}, mode: {config.pipeline_mode().value}"
)
split = config.pipeline_split()
if split is not None:
print(f" split: one of {len(split.paths)} paths per batch, p={list(split.p)}, paths={[list(path) for path in split.paths]}")

_record_config(json_path, os.path.join(self.output_folder, "transform_params_used_for_training.json"))

Expand Down
71 changes: 68 additions & 3 deletions smauglab/transforms/gpu/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
import copy
import functools
import warnings
from collections.abc import Sequence
from typing import Any, Protocol, Union
from collections.abc import Iterator, Sequence
from typing import TYPE_CHECKING, Any, Protocol, Union

import kornia.augmentation as K
import torch
Expand Down Expand Up @@ -32,6 +32,11 @@
from torch.distributions import RelaxedBernoulli
from torch.nn import Module

from smauglab.transforms.rng import shared_rand

if TYPE_CHECKING:
from smauglab.config import SplitSpec

try: # private to kornia, and the only part of the faster parameter draw that needs it
from kornia.augmentation.container.image import ImageSequentialBase, _get_new_batch_shape

Expand Down Expand Up @@ -138,7 +143,12 @@ def apply_transform_class(


class AugmentationSequentialCustom(AugmentationSequential):
"""Custom AugmentationSequential to handle masks augmentations."""
"""Custom AugmentationSequential to handle masks augmentations.

`split` makes groups of stages mutually exclusive (see `smauglab.config.SplitSpec`):
one path is drawn per call, and the stages of every other path are left out of that
call's forward sequence, so they neither draw parameters nor touch image or mask.
"""

def __init__(
self,
Expand All @@ -150,6 +160,7 @@ def __init__(
random_apply_weights: list[float] | None = None,
transformation_matrix_mode: str = "silent",
extra_args: dict[DataKey, dict[str, Any]] | None = None,
split: "SplitSpec | None" = None,
) -> None:
self._transform_matrix: Tensor | None
self._transform_matrices: list[Tensor | None] = []
Expand Down Expand Up @@ -198,6 +209,60 @@ def __init__(
self._transform_matrix = None
self.extra_args = extra_args or {DataKey.MASK: {"resample": Resample.NEAREST, "align_corners": None}}

self.split_spec = split
self._split_paths = None if split is None else self._resolve_split(split)

def _resolve_split(self, split: "SplitSpec") -> tuple[frozenset[str], ...]:
"""Each path as the names of the child modules it covers.

A config key is its transform's class name, so that is the match. Every name
must be exactly one stage here: a stage wrapped in a `RandomChooseX` bucket is
not a child, and gating something that is not there would make the split a
silent no-op.
"""
by_class: dict[str, list[str]] = {}
for child_name, child in self.named_children():
by_class.setdefault(type(child).__name__, []).append(child_name)
paths = []
for path in split.paths:
members = set()
for name in path:
found = by_class.get(name, [])
if len(found) != 1:
raise ValueError(
f"pipeline.split names {name!r}, which is {'not a stage' if not found else 'more than one stage'} of this pipeline"
)
members.add(found[0])
paths.append(frozenset(members))
return tuple(paths)

def _draw_skipped(self) -> frozenset[str]:
"""Draw this call's path; return the stages of every other path."""
assert self.split_spec is not None and self._split_paths is not None
# The shared draw, so DDP ranks take the same path for the same batch.
draw = float(shared_rand((1,), torch.device("cpu")).item())
taken, cumulative = len(self.split_spec.p) - 1, 0.0
for index, p in enumerate(self.split_spec.p):
cumulative += p
if draw < cumulative:
taken = index
break
return frozenset().union(*(path for index, path in enumerate(self._split_paths) if index != taken))

def get_forward_sequence(self, params: list[ParamItem] | None = None) -> Iterator[tuple[str, Module]]:
"""kornia's forward sequence, minus the stages of the paths not taken.

Only the parameter-drawing call (`params is None`) is filtered; `forward` then
iterates the parameters that call produced, so a skipped stage is skipped for
the image and the mask alike. Calls that replay `params` resolve stages from
them and are left alone.
"""
sequence = super().get_forward_sequence(params)
if params is not None or self._split_paths is None:
return sequence
skipped = self._draw_skipped()
return ((name, module) for name, module in sequence if name not in skipped)

def forward_parameters(self, batch_shape: torch.Size) -> list[ParamItem]:
"""kornia's own loop, with each leaf's draw routed through `_leaf_parameters`.

Expand Down
1 change: 1 addition & 0 deletions smauglab/transforms/gpu/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@ def __init__(self, json_path: str):
*transforms,
data_keys=["input", "mask"],
same_on_batch=True if config.same_on_batch() else None,
split=config.pipeline_split(),
)
Loading
Loading