From f948afdb21e8107294b6efb933f0eb789a12c462 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Fri, 16 Oct 2026 15:37:52 +0200 Subject: [PATCH 01/15] Serve the launcher with the docs and use it in the quick start, README and Linux page --- README.md | 9 ++++- doc/conf.py | 2 + doc/overview/CONTAINERS.md | 2 +- doc/overview/QUICKSTART.md | 75 ++++++++++++++--------------------- doc/overview/install/LINUX.md | 11 +++++ 5 files changed, 52 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 331775fb0..407cc883e 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,14 @@ The images we provide on [Docker Hub](https://hub.docker.com/r/deepmi/fastsurfer ### Usage All installation methods use the `run_fastsurfer.sh` call interface (replace the placeholder `` with [FastSurfer flags](doc/scripts/RUN_FASTSURFER.md#required-arguments)), which is the general starting point for FastSurfer. However, there are different ways to call this script depending on the installation, which we explain here: -1. For container installations, you need to set up the container (`` or ``) in addition to the ``: +1. For container installations, the [launcher](doc/overview/CONTAINERS.md#the-launcher) is the easiest way: it takes the `` and writes and runs the container command, with the image for your NVIDIA GPU (or the CPU): + + ```bash + curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py + python3 fastsurfer_container.py + ``` + + Written by hand, the container command needs the container's setup (`` or ``) in addition to the ``: 1. For __Singularity__, the syntax is ```text diff --git a/doc/conf.py b/doc/conf.py index 6fd5b186a..f26a6b3f5 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -239,6 +239,8 @@ def _tool_values_gitref(ref: str | None, *keys: str) -> tuple[str, ...]: # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "furo" html_static_path = ["_static"] +# served next to the pages, so the launcher downloads from deep-mi.org/FastSurfer// +html_extra_path = ["../tools/fastsurfer_container.py"] html_js_files = ["doc-version-link.js"] # points the announcement bar at the sibling doc tree html_title = project html_show_sphinx = False diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 4738085ca..4abb88672 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -37,7 +37,7 @@ The launcher Download it, and run FastSurfer: ```bash -curl -fsSLO https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer_container.py +curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py python3 fastsurfer_container.py \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ --sid subjectX --sd $HOME/my_fastsurfer_analysis \ diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index 9b5752fb2..f71e46199 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -12,17 +12,18 @@ a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 m `````{tab-set} -````{tab-item} Apptainer / Singularity +````{tab-item} Apptainer, Singularity or Docker +With [Apptainer, Singularity or Docker](install/LINUX.md) installed, the [launcher](CONTAINERS.md#the-launcher) +writes the container command for you: it picks the image for your NVIDIA GPU (or the CPU), downloads it on the first run, +and shares this directory with the container. + ```bash # 0. Create a directory for this test mkdir fastsurfer_test cd fastsurfer_test -# 1. Build the Apptainer image from our Docker image (only the first time) -mkdir -p $HOME/my_singularity_images -singularity build \ - $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ - docker://deepmi/fastsurfer:{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }} +# 1. Download the launcher (only the first time) +curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py # 2. Download an example brain MRI (if you don't have your own) # If you have your own, copy it to this directory and adjust @@ -32,38 +33,12 @@ curl -k \ -o "./140_orig.mgz" # 3. Run FastSurfer (full brain segmentation only) -singularity exec --nv \ - --no-mount home,cwd -e \ - -B "$PWD" \ - $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ - /fastsurfer/run_fastsurfer.sh \ - --t1 "$PWD/140_orig.mgz" \ - --sid subjectX --sd "$PWD" \ - --seg_only --no_biasfield --no_cereb --no_hypothal +python3 fastsurfer_container.py \ + --t1 140_orig.mgz --sid subjectX --sd . \ + --seg_only --no_biasfield --no_cereb --no_hypothal ``` -```` -````{tab-item} Docker -```bash -# 0. Create a directory for this test -mkdir fastsurfer_test -cd fastsurfer_test - -# 1. Download an example brain MRI (if you don't have your own) -# If you have your own, copy it to this directory and adjust -# the filename after --t1 below. -curl -k \ - https://surfer.nmr.mgh.harvard.edu/pub/data/tutorial_data/buckner_data/tutorial_subjs/140/mri/orig.mgz \ - -o "./140_orig.mgz" - -# 2. Run FastSurfer (full brain segmentation only) -docker run --gpus all -v "$PWD:$PWD" \ - --rm --user $(id -u):$(id -g) \ - deepmi/fastsurfer:{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }} \ - --t1 "$PWD/140_orig.mgz" \ - --sid subjectX --sd "$PWD" \ - --seg_only --no_biasfield --no_cereb --no_hypothal -``` +[Running FastSurfer in a container](CONTAINERS.md) shows the `singularity` and `docker` commands the launcher runs. ```` ````{tab-item} macOS package @@ -94,21 +69,31 @@ run_fastsurfer.sh --t1 "$PWD/140_orig.mgz" \ That's it, it will run the full brain segmentation. For speed, we switched off the cerebellum and hypothalamic sub-segmentation (would add a couple minutes). We also switched off the bias field correction, which is used to compute partial volume estimates for the statsfiles, so you might want to switch it on again if you want the volume statistics text file (under ```subjectX/stats```). Also if you need the estimated total intracranial volume for correcting the stats, you would either need to run the surface stream or switch on the Talairach registration with -`--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license) and pass it into the container, see [Running FastSurfer in a container](CONTAINERS.md). +`--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license): pass it with `--fs_license `, or set the environment variable `FS_LICENSE` to its path (the launcher passes it on to the container). + +You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer, in the image the launcher used (it names the image when it starts, e.g. `{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}`). With Apptainer or Singularity: -You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer, for example with the Singularity image (in the macOS console, call `nib-convert` directly): +```bash +# Convert mgz to nifti +singularity exec --no-mount home,cwd -e -B "$PWD" \ + $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ + nib-convert "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ + "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" +``` + +With Docker: ```bash # Convert mgz to nifti -singularity exec --nv \ - --no-mount home -e \ - -B "$PWD" \ - $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ - nib-convert "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ - "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" +docker run --rm --user $(id -u):$(id -g) -v "$PWD:$PWD" --entrypoint nib-convert \ + deepmi/fastsurfer:{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }} \ + "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ + "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" ``` -and find the segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz```. If you have FreeSurfer installed, just use FreeView to look at the result (or really any other image viewer): +In the macOS console, call `nib-convert` directly with the same two paths. + +Then find the segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz```. If you have FreeSurfer installed, just use FreeView to look at the result (or really any other image viewer): ```bash # FreeView diff --git a/doc/overview/install/LINUX.md b/doc/overview/install/LINUX.md index 9fe57d288..8498ae3eb 100644 --- a/doc/overview/install/LINUX.md +++ b/doc/overview/install/LINUX.md @@ -9,6 +9,17 @@ and for the CPU only, on [Docker Hub](https://hub.docker.com/r/deepmi/fastsurfer A GPU with enough memory makes the segmentation much faster, see the [system requirements](../intro.rst#system-requirements). Without a GPU, use the [CPU image](#cpu-only). +With Apptainer, Singularity or Docker installed, the [launcher](../CONTAINERS.md#the-launcher) does the rest: it +picks the image for your NVIDIA GPU and driver (or the CPU image), downloads it, and runs FastSurfer with the flags of +`run_fastsurfer.sh`: + +```bash +curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py +python3 fastsurfer_container.py --t1 --sid --sd +``` + +The sections below explain the images and how to download them by hand. + The images are tagged `deepmi/fastsurfer:-v`, where `` is `cu` for NVIDIA GPUs (see [below](#nvidia-gpus)), `rocm` for AMD GPUs (experimental), or `cpu` without GPU support (smaller and thus faster to download), and `` is the FastSurfer version. `latest` points to the newest NVIDIA image, From 878521361dda26991621e88fe4152dbcede29140 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Fri, 16 Oct 2026 16:47:21 +0200 Subject: [PATCH 02/15] Send only Apple silicon Macs to the package in the quick start and require a container runtime for the others --- doc/overview/QUICKSTART.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index f71e46199..fb2f2cc77 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -1,8 +1,9 @@ Quick Start =========== -This example segments one brain MRI in a few minutes. Choose how you run FastSurfer: in a container on Linux -(Apptainer, Singularity or Docker), with the [macOS package](install/MACOS.md), or without installing anything in -[Google Colab](#google-colab). Other systems are covered in the [installation guide](INSTALL.md). +This example segments one brain MRI in a few minutes. Choose how you run FastSurfer: on a Mac with Apple silicon +(M1 or newer) with the [macOS package](install/MACOS.md), on Linux, Windows or a Mac with an Intel CPU in a container +(Apptainer, Singularity or Docker), or without installing anything in [Google Colab](#google-colab). The +[installation guide](INSTALL.md) helps to choose. Segment an example image ------------------------ @@ -12,10 +13,13 @@ a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 m `````{tab-set} -````{tab-item} Apptainer, Singularity or Docker -With [Apptainer, Singularity or Docker](install/LINUX.md) installed, the [launcher](CONTAINERS.md#the-launcher) -writes the container command for you: it picks the image for your NVIDIA GPU (or the CPU), downloads it on the first run, -and shares this directory with the container. +````{tab-item} Linux, Windows or Intel Mac +This needs a working installation of Apptainer, Singularity or Docker, which these commands do not set up: see +[Linux](install/LINUX.md), [Windows](install/WINDOWS.md) (Docker in WSL2, run the commands there) or +[Intel Macs](install/MACOS.md#docker-intel-macs). `docker run --rm hello-world` or `apptainer --version` checks it. + +The [launcher](CONTAINERS.md#the-launcher) then writes the container command for you: it picks the image for your +NVIDIA GPU (or the CPU), downloads it on the first run, and shares this directory with the container. ```bash # 0. Create a directory for this test @@ -41,9 +45,9 @@ python3 fastsurfer_container.py \ [Running FastSurfer in a container](CONTAINERS.md) shows the `singularity` and `docker` commands the launcher runs. ```` -````{tab-item} macOS package -[Install the package](install/MACOS.md), open the FastSurfer app from your Applications folder, and run these -commands in the Terminal window it opens: +````{tab-item} Mac with Apple silicon +On a Mac with Apple silicon (M1 or newer), [install the macOS package](install/MACOS.md), open the FastSurfer app +from your Applications folder, and run these commands in the Terminal window it opens: ```bash # 0. Create a directory for this test From e8da3cc2e95df3ff114ce13208aa4e81e4350918 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Fri, 16 Oct 2026 17:46:27 +0200 Subject: [PATCH 03/15] Show the nifti conversion in tabs for Apptainer, Docker and the macOS package --- doc/overview/QUICKSTART.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index fb2f2cc77..dd510221b 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -75,8 +75,11 @@ We also switched off the bias field correction, which is used to compute partial Also if you need the estimated total intracranial volume for correcting the stats, you would either need to run the surface stream or switch on the Talairach registration with `--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license): pass it with `--fs_license `, or set the environment variable `FS_LICENSE` to its path (the launcher passes it on to the container). -You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer, in the image the launcher used (it names the image when it starts, e.g. `{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}`). With Apptainer or Singularity: +You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer, in the image the launcher used (it names the image when it starts, e.g. `{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}`): +`````{tab-set} + +````{tab-item} Apptainer / Singularity ```bash # Convert mgz to nifti singularity exec --no-mount home,cwd -e -B "$PWD" \ @@ -84,9 +87,9 @@ singularity exec --no-mount home,cwd -e -B "$PWD" \ nib-convert "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" ``` +```` -With Docker: - +````{tab-item} Docker ```bash # Convert mgz to nifti docker run --rm --user $(id -u):$(id -g) -v "$PWD:$PWD" --entrypoint nib-convert \ @@ -94,8 +97,17 @@ docker run --rm --user $(id -u):$(id -g) -v "$PWD:$PWD" --entrypoint nib-convert "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" ``` +```` -In the macOS console, call `nib-convert` directly with the same two paths. +````{tab-item} Mac with Apple silicon +```bash +# Convert mgz to nifti, in the Terminal window of the FastSurfer app +nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz \ + subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz +``` +```` + +````` Then find the segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz```. If you have FreeSurfer installed, just use FreeView to look at the result (or really any other image viewer): From 1f75b9aa3930069dd8aded4e503f83f61e0e52ee Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Fri, 16 Oct 2026 18:31:34 +0200 Subject: [PATCH 04/15] Add --long and --exec to the launcher, and use the system certificates when Python has none --- doc/overview/CONTAINERS.md | 15 +- doc/overview/QUICKSTART.md | 29 ++-- test/utils/test_fastsurfer_container.py | 68 ++++++++-- tools/fastsurfer_container.py | 173 +++++++++++++++++++----- 4 files changed, 222 insertions(+), 63 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 4abb88672..82764faef 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -60,7 +60,20 @@ them on the CPU. The image of an older release may still support that GPU, but t recommends the newest release on the CPU, or a machine with a newer GPU, over an old version. With `--brun`, it runs `brun_fastsurfer.sh` for several subjects; the paths in a subjects list have -to be absolute. `--print_only` prints the commands instead of running them, `--update_launcher` +to be absolute. With `--long`, it runs `long_fastsurfer.sh` for the time points of one person (see +[long_fastsurfer.sh](../scripts/long_fastsurfer.rst)). + +`--exec ` runs any other program of the image, for example to convert a segmentation to NIfTI: + +```bash +python3 fastsurfer_container.py --exec nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz aparc.nii.gz +``` + +It comes last, as all arguments after the program go to the program. The program runs in the current +directory, which the launcher shares with the container, as it does the directories of absolute paths +in the arguments. Relative paths therefore only work inside the current directory. + +`--print_only` prints the commands instead of running them, `--update_launcher` replaces the launcher with the current one, and `--help` lists all launcher options. The sections below explain the commands it runs. diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index dd510221b..d5c31cd42 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -14,6 +14,8 @@ a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 m `````{tab-set} ````{tab-item} Linux, Windows or Intel Mac +:sync: container + This needs a working installation of Apptainer, Singularity or Docker, which these commands do not set up: see [Linux](install/LINUX.md), [Windows](install/WINDOWS.md) (Docker in WSL2, run the commands there) or [Intel Macs](install/MACOS.md#docker-intel-macs). `docker run --rm hello-world` or `apptainer --version` checks it. @@ -46,6 +48,8 @@ python3 fastsurfer_container.py \ ```` ````{tab-item} Mac with Apple silicon +:sync: macos + On a Mac with Apple silicon (M1 or newer), [install the macOS package](install/MACOS.md), open the FastSurfer app from your Applications folder, and run these commands in the Terminal window it opens: @@ -75,31 +79,22 @@ We also switched off the bias field correction, which is used to compute partial Also if you need the estimated total intracranial volume for correcting the stats, you would either need to run the surface stream or switch on the Talairach registration with `--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license): pass it with `--fs_license `, or set the environment variable `FS_LICENSE` to its path (the launcher passes it on to the container). -You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer, in the image the launcher used (it names the image when it starts, e.g. `{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}`): +You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer: `````{tab-set} -````{tab-item} Apptainer / Singularity -```bash -# Convert mgz to nifti -singularity exec --no-mount home,cwd -e -B "$PWD" \ - $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ - nib-convert "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ - "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" -``` -```` - -````{tab-item} Docker +````{tab-item} Linux, Windows or Intel Mac +:sync: container ```bash -# Convert mgz to nifti -docker run --rm --user $(id -u):$(id -g) -v "$PWD:$PWD" --entrypoint nib-convert \ - deepmi/fastsurfer:{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }} \ - "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ - "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" +# Convert mgz to nifti, in the image the launcher used for FastSurfer +python3 fastsurfer_container.py --exec nib-convert \ + subjectX/mri/aparc.DKTatlas+aseg.deep.mgz \ + subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz ``` ```` ````{tab-item} Mac with Apple silicon +:sync: macos ```bash # Convert mgz to nifti, in the Terminal window of the FastSurfer app nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz \ diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 07539e205..c5c121f4e 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -386,6 +386,45 @@ def test_plan_paths_brun(tmp_path): launcher.plan_paths(["--subjects_list", str(subjects)], tmp_path) +def test_plan_paths_long(tmp_path): + # long_fastsurfer.sh: several T1s (and T2s) after one flag, one per time point + data = tmp_path / "data" + data.mkdir() + for name in ("tp1.mgz", "tp2.mgz", "t2_1.mgz", "t2_2.mgz"): + (data / name).write_bytes(b"") + args = ["--tid", "person", "--t1s", "data/tp1.mgz", str(data / "tp2.mgz"), "--tpids", "tp1", "tp2", + "--t2s", "data/t2_1.mgz", "data/t2_2.mgz", "--sd", "out", "--log", "logs/long.log"] + out, mounts = launcher.plan_paths(args, tmp_path) + assert out == ["--tid", "person", "--t1s", str(data / "tp1.mgz"), str(data / "tp2.mgz"), "--tpids", "tp1", "tp2", + "--t2s", str(data / "t2_1.mgz"), str(data / "t2_2.mgz"), "--sd", str(tmp_path / "out"), + "--log", str(tmp_path / "logs/long.log")] + assert set(launcher.minimal_mounts(mounts)) == {data, tmp_path / "out", tmp_path / "logs"} + # every time point has to exist, not only the first + with pytest.raises(launcher.LauncherError, match="tp3.mgz"): + launcher.plan_paths(["--t1s", "data/tp1.mgz", "data/tp3.mgz", "--sd", "out"], tmp_path) + + +def test_exec_mounts(tmp_path): + work, data, results = tmp_path / "work", tmp_path / "data", tmp_path / "results" + for directory in (work, data, results): + directory.mkdir() + (data / "in.mgz").write_bytes(b"") + args = ["-v", str(data / "in.mgz"), f"--out={results / 'new.nii.gz'}", "relative.mgz", "2", + str(tmp_path / "missing" / "x.mgz")] + mounts = launcher.minimal_mounts(launcher.exec_mounts(args, work)) + # the current directory, the directory of an input, and that of an output in --flag=path form; a path + # whose directory does not exist is not taken for one, and relative paths stay in the current directory + assert mounts == sorted([work, data, results]) + + +def test_split_exec(): + argv, program_args = launcher.split_exec(["--runtime", "docker", "--exec", "mri_convert", "--help", "--image", "x"]) + options, rest = launcher.make_parser().parse_known_args(argv) + # the program's own flags, even the launcher's --help and --image, reach the program + assert (options.runtime, options.exec, rest) == ("docker", "mri_convert", []) + assert program_args == ["--help", "--image", "x"] + + def test_link_directories(tmp_path): # a chain of links, as in a DataLad dataset: the link, a relative link in another directory, the file for name in ("data", "annex/key", "objects/abc"): @@ -438,7 +477,8 @@ def test_build_image(tmp_path): def test_commands(tmp_path): args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] - docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") + run = launcher.RUN_SCRIPT + docker = launcher.docker_command("img:tag", True, [tmp_path], run, args, [], "1000:1000") # --init, so Ctrl-C and a cancelled job reach FastSurfer init = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1"] # the host's time zone, on Linux @@ -447,18 +487,28 @@ def test_commands(tmp_path): assert docker == init + ["--user", "1000:1000", "--gpus", "all", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID"] + zone + [ "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args # selected GPUs, quoted so Docker does not split the list - selected = launcher.docker_command("img:tag", True, [], "run_fastsurfer.sh", [], [], "1:1", devices=[1, 3]) + selected = launcher.docker_command("img:tag", True, [], run, [], [], "1:1", devices=[1, 3]) assert selected[8:10] == ["--gpus", '"device=1,3"'] - sif_gpu = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], "run_fastsurfer.sh", [], [], - devices=[2]) + sif_gpu = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], run, [], [], devices=[2]) assert sif_gpu[2:8] == ["--nv", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", "--env", "CUDA_VISIBLE_DEVICES=2", "--no-mount"] - emulated = launcher.docker_command("img:tag", False, [tmp_path], "run_fastsurfer.sh", args, [], "501:20", True) + emulated = launcher.docker_command("img:tag", False, [tmp_path], run, args, [], "501:20", True) assert emulated[:10] == init + ["--user", "501:20", "--platform", "linux/amd64"] - brun = launcher.docker_command("img:tag", False, [tmp_path], "brun_fastsurfer.sh", args, [], "1000:1000") - assert brun[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", "img:tag", - "/fastsurfer/brun_fastsurfer.sh"] + # the other scripts and programs run through the image's entrypoint, which sets up the environment + for program in (launcher.SCRIPTS["brun"], launcher.SCRIPTS["long"], "nib-convert"): + other = launcher.docker_command("img:tag", False, [tmp_path], program, args, [], "1000:1000") + assert other[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", + "img:tag", program] sif = tmp_path / "fs.sif" - apptainer = launcher.apptainer_command("apptainer", sif, False, [tmp_path], "run_fastsurfer.sh", args, []) + apptainer = launcher.apptainer_command("apptainer", sif, False, [tmp_path], run, args, []) assert apptainer == ["apptainer", "exec", "--no-mount", "home,cwd", "-e", "-B", str(tmp_path), str(sif), "/fastsurfer/run_fastsurfer.sh"] + args + # --exec: in the current directory, and with Docker attached to the terminal if there is one + exec_args = ["in.mgz", "out.nii.gz"] + in_dir = launcher.apptainer_command("apptainer", sif, False, [tmp_path], "nib-convert", exec_args, [], + workdir=tmp_path) + assert in_dir[-6:] == ["--pwd", str(tmp_path), str(sif), "nib-convert"] + exec_args + docker_exec = launcher.docker_command("img:tag", False, [tmp_path], "bash", [], [], "1:1", workdir=tmp_path, + interactive=True) + assert docker_exec[8] == "-it" + assert docker_exec[docker_exec.index("-w") + 1] == str(tmp_path) diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 746f428cd..836232b44 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -16,11 +16,15 @@ """ Run FastSurfer in its container image, without writing the container command. -Takes the flags of run_fastsurfer.sh (or, with --brun, of brun_fastsurfer.sh) and runs them in the -official image with Apptainer, Singularity or Docker, whichever is installed: +Takes the flags of run_fastsurfer.sh (or, with --brun, of brun_fastsurfer.sh, with --long, of +long_fastsurfer.sh) and runs them in the official image with Apptainer, Singularity or Docker, +whichever is installed: python3 fastsurfer_container.py --t1 data/t1.nii.gz --sd results --sid subject1 +--exec runs any other program of the image, e.g. --exec nib-convert in.mgz out.nii.gz, in the current +directory. + It runs the FastSurfer release image installed on the machine (the newest release if there is none, or the image --image names), picks the image type that fits the NVIDIA GPU and driver (or the CPU image), downloads it once, @@ -68,13 +72,18 @@ # flags of run_fastsurfer.sh and brun_fastsurfer.sh whose value is a file that has to exist INPUT_FLAGS = {"--t1", "--t2", "--lesion_mask", "--fs_license", "--subjects_list", "--subject_list"} +# flags of long_fastsurfer.sh followed by several files that have to exist, one per time point +MULTI_INPUT_FLAGS = {"--t1s", "--t2s"} # flags whose value is a file or directory FastSurfer writes, so its directory is created if missing OUTPUT_FLAGS = { "--sd", "--seg_log", "--conformed_name", "--norm_name", "--norm_name_t2", "--seg", "--asegdkt_segfile", "--aparc_aseg_segfile", "--asegdkt_statsfile", "--aseg_statsfile", "--aseg_segfile", "--mask_name", "--cereb_segfile", "--cereb_statsfile", "--hypo_segfile", - "--hypo_statsfile", "--statusfile", + "--hypo_statsfile", "--statusfile", "--log", } +# the FastSurfer scripts the launcher runs: the default, and those of --brun and --long +RUN_SCRIPT = "/fastsurfer/run_fastsurfer.sh" +SCRIPTS = {"brun": "/fastsurfer/brun_fastsurfer.sh", "long": "/fastsurfer/long_fastsurfer.sh"} class LauncherError(Exception): @@ -429,6 +438,26 @@ def image_reference(prefix: str, version: str) -> str: SIF_PREFIX = DOCKER_REPO.rsplit("/", 1)[-1] + "-" +# the root certificates macOS ships; Python from python.org has none of its own until its "Install +# Certificates" script runs, and fails every HTTPS request until then +SYSTEM_CA_FILE = "/etc/ssl/cert.pem" + + +def open_url(url: str): + """urllib's urlopen, retried with the system's certificates if Python cannot verify the server.""" + import ssl + import urllib.error + import urllib.request + + try: + return urllib.request.urlopen(url, timeout=HUB_TIMEOUT) + except urllib.error.URLError as error: + if not isinstance(error.reason, ssl.SSLCertVerificationError) or not os.path.isfile(SYSTEM_CA_FILE): + raise + context = ssl.create_default_context(cafile=SYSTEM_CA_FILE) + return urllib.request.urlopen(url, timeout=HUB_TIMEOUT, context=context) + + def hub_tags() -> dict[str, set[str]] | None: """ The release tags on Docker Hub and the digests each is known by, None if Docker Hub cannot be reached. @@ -438,13 +467,12 @@ def hub_tags() -> dict[str, set[str]] | None: recorded. """ import json - import urllib.request url = f"https://hub.docker.com/v2/repositories/{DOCKER_REPO}/tags?page_size=100&name=-v" tags: dict[str, set[str]] = {} try: while url and len(tags) < 2000: - with urllib.request.urlopen(url, timeout=HUB_TIMEOUT) as response: + with open_url(url) as response: page = json.load(response) for result in page.get("results", []): digests = {result.get("digest")} | {image.get("digest") for image in result.get("images", [])} @@ -756,7 +784,7 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st Parameters ---------- args : list[str] - The arguments for run_fastsurfer.sh or brun_fastsurfer.sh. + The arguments for run_fastsurfer.sh, brun_fastsurfer.sh or long_fastsurfer.sh. cwd : Path The directory relative paths are relative to. create : bool, default=True @@ -785,6 +813,15 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st mounts.extend(_subjects_list_mounts(path, create)) out.append(str(path)) i += 1 + elif flag in MULTI_INPUT_FLAGS: + # long_fastsurfer.sh: one path per time point, until the next flag + while i < len(args) and not args[i].startswith("-"): + path = _absolute(args[i], cwd) + if not path.exists(): + raise LauncherError(f"{flag} {args[i]}: no such file.") + mounts.extend(_link_directories(path)) + out.append(str(path)) + i += 1 elif flag == "--subjects": # brun_fastsurfer.sh: = ... until the next flag while i < len(args) and not args[i].startswith("--"): @@ -800,6 +837,27 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st return out, mounts +def exec_mounts(args: list[str], cwd: Path) -> list[Path]: + """ + The directories to share for any program: the current directory, where it runs, and those of its + absolute paths, also in `--flag=/path` form. + + An absolute path that exists is shared like an input, one whose directory exists like an output; + any other argument is not taken for a path. Relative paths only work inside the current directory. + """ + mounts = [cwd] + for arg in args: + value = arg.partition("=")[2] if arg.startswith("-") else arg + if not os.path.isabs(value): + continue + path = Path(os.path.normpath(value)) + if path.exists(): + mounts.extend(_link_directories(path)) + elif path.parent.is_dir(): + mounts.append(path.parent) + return mounts + + def _subjects_list_mounts(subjects_list: Path, create: bool = True) -> list[Path]: """ The directories to share for a brun_fastsurfer.sh subjects list, whose paths must be absolute. @@ -863,9 +921,13 @@ def runtime_too_old(version_text: str) -> str | None: CUDA_ORDER = "CUDA_DEVICE_ORDER=PCI_BUS_ID" -def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], script: str, - args: list[str], extra: list[str], devices: list[int] | None = None) -> list[str]: - """The Apptainer/Singularity command, showing only `devices` if given; the home directory is not mounted.""" +def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], program: str, + args: list[str], extra: list[str], devices: list[int] | None = None, + workdir: Path | None = None) -> list[str]: + """ + The Apptainer/Singularity command running `program`, showing only `devices` if given and starting in + `workdir` if given; the home directory is not mounted. + """ cmd = [runtime, "exec"] if gpu: cmd += ["--nv", "--env", CUDA_ORDER] @@ -874,20 +936,25 @@ def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], sc cmd += ["--no-mount", "home,cwd", "-e"] for mount in mounts: cmd += ["-B", str(mount)] - return cmd + extra + [str(sif), f"/fastsurfer/{script}"] + args + if workdir is not None: + cmd += ["--pwd", str(workdir)] + return cmd + extra + [str(sif), program] + args -def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: list[str], +def docker_command(image: str, gpu: bool, mounts: list[Path], program: str, args: list[str], extra: list[str], user: str, emulate: bool = False, devices: list[int] | None = None, - ) -> list[str]: + workdir: Path | None = None, interactive: bool = False) -> list[str]: """ - The Docker command, running as the calling user and showing only `devices` if given. + The Docker command running `program`, as the calling user, showing only `devices` if given and + starting in `workdir` if given. - `emulate` runs the Intel image on Apple silicon. + `emulate` runs the Intel image on Apple silicon; `interactive` connects the terminal, e.g. for a shell. """ # --init: the image's entrypoint is a shell, which as the first process ignores Ctrl-C and the # signal a cancelled cluster job sends; tini passes them to all of FastSurfer's processes instead cmd = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1", "--user", user] + if interactive: + cmd += ["-it"] if gpu: # the quotes are part of the value: without them, Docker splits a list of devices at the comma selection = "all" if devices is None else f'"device={",".join(map(str, devices))}"' @@ -899,10 +966,12 @@ def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: cmd += ["-v", "/etc/localtime:/etc/localtime:ro"] for mount in mounts: cmd += ["-v", f"{mount}:{mount}"] + if workdir is not None: + cmd += ["-w", str(workdir)] cmd += extra - if script != "run_fastsurfer.sh": - # the image's entrypoint runs run_fastsurfer.sh; keep its environment setup for the other scripts - return cmd + ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", image, f"/fastsurfer/{script}"] + args + if program != RUN_SCRIPT: + # the image's entrypoint runs run_fastsurfer.sh; keep its environment setup for other programs + return cmd + ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", image, program] + args return cmd + [image] + args @@ -992,10 +1061,8 @@ def build_image(command: list[str], anonymous: bool = False) -> bool: def stable_launcher() -> bytes | None: """The launcher on the stable branch, which knows the images of the newest release; None if unreachable.""" - import urllib.request - try: - with urllib.request.urlopen(LAUNCHER_URL, timeout=HUB_TIMEOUT) as response: + with open_url(LAUNCHER_URL) as response: return response.read() except OSError: return None @@ -1032,8 +1099,8 @@ def make_parser() -> argparse.ArgumentParser: prog="fastsurfer_container.py", description=__doc__.split("\n\n")[0].strip(), epilog=( - "All other arguments are passed to run_fastsurfer.sh (or brun_fastsurfer.sh), see " - "https://deep-mi.org/FastSurfer/stable/scripts/RUN_FASTSURFER.html. Relative paths are " + "All other arguments are passed to run_fastsurfer.sh (or brun_fastsurfer.sh, long_fastsurfer.sh), " + "see https://deep-mi.org/FastSurfer/stable/scripts/RUN_FASTSURFER.html. Relative paths are " "relative to the current directory." ), allow_abbrev=False, @@ -1055,19 +1122,43 @@ def make_parser() -> argparse.ArgumentParser: default=Path(os.environ.get("FASTSURFER_IMAGE_DIR", "~/my_singularity_images")).expanduser(), help="where Apptainer/Singularity images are kept (default: $FASTSURFER_IMAGE_DIR or " "~/my_singularity_images)") - parser.add_argument("--brun", action="store_true", help="run brun_fastsurfer.sh to process several subjects") + script = parser.add_mutually_exclusive_group() + script.add_argument("--brun", action="store_true", help="run brun_fastsurfer.sh to process several subjects") + script.add_argument("--long", action="store_true", + help="run long_fastsurfer.sh for the time points of one person (longitudinal processing)") + script.add_argument("--exec", metavar="PROGRAM", + help="run another program of the image instead, e.g. --exec nib-convert in.mgz out.nii.gz. " + "It comes last: all later arguments go to the program. It runs in the current " + "directory, which is shared with the container, as are the directories of absolute " + "paths in its arguments") parser.add_argument("--runtime_args", default="", help="additional arguments for the container runtime, e.g. --runtime_args '--group-add 1001'") parser.add_argument("--print_only", action="store_true", help="print the commands instead of running them") return parser +def split_exec(argv: list[str]) -> tuple[list[str], list[str] | None]: + """ + The launcher's arguments up to `--exec `, and the program's arguments after it (None without + --exec), so that its flags, e.g. --help, reach the program rather than the launcher. + """ + if "--exec" not in argv: + return argv, None + k = argv.index("--exec") + return argv[:k + 2], argv[k + 2:] + + def main(argv: list[str] | None = None) -> int: parser = make_parser() + argv, program_args = split_exec(sys.argv[1:] if argv is None else argv) options, fastsurfer_args = parser.parse_known_args(argv) if options.update_launcher: return update_launcher() - if not fastsurfer_args: + if options.exec is not None: + if fastsurfer_args: + parser.error(f"unrecognized arguments before --exec: {' '.join(fastsurfer_args)}") + fastsurfer_args = program_args + elif not fastsurfer_args: parser.print_help() return 1 @@ -1098,8 +1189,11 @@ def info(message: str, level: str = "INFO") -> None: raise LauncherError("--brun needs the subjects in --subjects_list or --subjects = ...") cwd = Path.cwd() - fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) - args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) + if options.exec is not None: + args, mounts = fastsurfer_args, exec_mounts(fastsurfer_args, cwd) + else: + fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) + args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) mounts = minimal_mounts(mounts) if Path("/") in mounts: raise LauncherError("The paths would share the whole file system with the container, keep the " @@ -1107,19 +1201,20 @@ def info(message: str, level: str = "INFO") -> None: if Path.home() in mounts: info("This shares your home directory with the container. Settings in it may affect " "FastSurfer, so keep the data in a subdirectory.") - if "--fs_license" not in args and "--seg_only" not in args: + if options.exec is None and "--fs_license" not in args and "--seg_only" not in args: info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") - # the GPUs and the image type + # the GPUs and the image type; another program's flags say nothing about them kind, value = parse_image_spec(options.image) gpus = query_nvidia_gpus() + request = [] if options.exec is not None else args # whether the newest images support none of the GPUs, so they run on the CPU here gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None - wants_gpu = gpu_request(args, options.brun)[0] != "cpu" and bool(gpus) + wants_gpu = gpu_request(request, options.brun)[0] != "cpu" and bool(gpus) if kind in ("auto", "version"): - build = plan_build(args, gpus, options.brun) + build = plan_build(request, gpus, options.brun) elif kind in ("tag", "release"): - build = plan_build_for_type(args, gpus, options.brun, RELEASE_TAG.fullmatch(value).group(1)) + build = plan_build_for_type(request, gpus, options.brun, RELEASE_TAG.fullmatch(value).group(1)) else: build = Build("custom", wants_gpu) @@ -1165,7 +1260,11 @@ def info(message: str, level: str = "INFO") -> None: for note in build.notes: info(note) - script = "brun_fastsurfer.sh" if options.brun else "run_fastsurfer.sh" + if options.exec is not None: + program, workdir = options.exec, cwd + else: + program = SCRIPTS["brun"] if options.brun else SCRIPTS["long"] if options.long else RUN_SCRIPT + workdir = None extra = shlex.split(options.runtime_args) commands = [] if runtime == "docker": @@ -1174,8 +1273,10 @@ def info(message: str, level: str = "INFO") -> None: emulate = apple_silicon and (kind != "docker" or value.startswith(DOCKER_REPO)) if plan.pull: commands.append(["docker", "pull"] + (["--platform", "linux/amd64"] if emulate else []) + [plan.image]) - commands.append(docker_command(plan.image, build.gpu, mounts, script, args, extra, user, emulate, - build.devices)) + # Apptainer passes the terminal on itself; Docker only when asked + interactive = options.exec is not None and sys.stdin.isatty() and sys.stdout.isatty() + commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, + build.devices, workdir, interactive)) if not options.print_only: memory = docker_memory_gib() if memory < MIN_DOCKER_MEMORY_GIB: @@ -1188,8 +1289,8 @@ def info(message: str, level: str = "INFO") -> None: raise LauncherError(f"--image {value}: no such file.") if plan.build_from is not None: commands.append([runtime, "build", plan.image, plan.build_from]) - commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, script, args, extra, - build.devices)) + commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, program, args, extra, + build.devices, workdir)) except LauncherError as error: print(f"ERROR: {error}", file=sys.stderr) return 1 From 2bd08661ed468b02f4e24dc19d75d4083a6548b7 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Fri, 16 Oct 2026 18:34:39 +0200 Subject: [PATCH 05/15] Count only the images of the runtime named with --runtime, and let --exec use the newest of several --- doc/overview/CONTAINERS.md | 3 ++- test/utils/test_fastsurfer_container.py | 9 +++++++++ tools/fastsurfer_container.py | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 82764faef..ad5bd4b5c 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -71,7 +71,8 @@ python3 fastsurfer_container.py --exec nib-convert subjectX/mri/aparc.DKTatlas+a It comes last, as all arguments after the program go to the program. The program runs in the current directory, which the launcher shares with the container, as it does the directories of absolute paths -in the arguments. Relative paths therefore only work inside the current directory. +in the arguments. Relative paths therefore only work inside the current directory. If several FastSurfer +images are installed, `--exec` uses the newest instead of asking. `--print_only` prints the commands instead of running them, `--update_launcher` replaces the launcher with the current one, and `--help` lists all launcher options. The sections diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index c5c121f4e..32d9f35ae 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -294,6 +294,15 @@ def test_decide_image_several_installed(tmp_path): plan = decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}, update=True) assert plan.build_from is not None and plan.tag == "cu132-v2.6.0" assert any("docker image rm deepmi/fastsurfer:cu132-v2.5.4" in note for _, note in plan.notes) + # a runtime named with --runtime counts only its own images while it has one + assert decide(tmp_path, runtime="apptainer", docker={"cu132-v2.5.4": "sha256:b"}, + sif={"cu132-v2.6.0": ""}).tag == "cu132-v2.6.0" + assert decide(tmp_path, runtime="docker", docker={"cu132-v2.5.4": "sha256:b"}, + sif={"cu132-v2.6.0": ""}).tag == "cu132-v2.5.4" + # a program other than FastSurfer (--exec) takes the newest instead of stopping + plan = launcher.decide_image("auto", "", "cu132", "auto", BOTH, {"cu132-v2.5.4": "sha256:b"}, {"cu132-v2.6.0": ""}, + HUB, tmp_path, False, newest_of_several=True) + assert plan.tag == "cu132-v2.6.0" and any("uses the newest" in note for _, note in plan.notes) def test_decide_image_digest(tmp_path): diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 836232b44..b17c46d5f 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -590,7 +590,8 @@ def _apptainer_runtime(runtime: str, installed: set[str]) -> str | None: def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: set[str], docker_images: dict[str, str] | None, sif_images: dict[str, str], hub: dict[str, set[str]] | None, image_dir: Path, update: bool, - local_id: str | None = None, gpus: list[Gpu] | None = None) -> ImagePlan: + local_id: str | None = None, gpus: list[Gpu] | None = None, + newest_of_several: bool = False) -> ImagePlan: """ Which image to run, with which runtime, and whether it has to be downloaded or built first. @@ -625,6 +626,9 @@ def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: se gpus : list[Gpu], None The GPUs the run will use, to tell which installed images run here; empty for a run on the CPU. None counts only images of type `prefix`. + newest_of_several : bool, default=False + Use the newest of several installed images instead of stopping, for a program other than + FastSurfer, whose results do not depend on the image. """ apptainer = _apptainer_runtime(runtime, installed) docker_images = docker_images or {} @@ -703,6 +707,10 @@ def runs_here(image_type: str) -> bool: candidates = sorted({t for t in [*docker_images, *sif_images] if (m := RELEASE_TAG.fullmatch(t)) and runs_here(m.group(1))}, key=lambda t: (parse_version(t.rsplit("-v", 1)[-1]), t)) + # a runtime named with --runtime counts its own images, and the others only if it has none + own_store = {"docker": docker_images, "apptainer": sif_images, "singularity": sif_images}.get(runtime) + if own_store is not None and any(t in own_store for t in candidates): + candidates = [t for t in candidates if t in own_store] newest = newest_version(list(hub), prefix) if hub is not None else None if update or not candidates: if newest is None: @@ -725,6 +733,11 @@ def runs_here(image_type: str) -> bool: if newest is not None and parse_version(newest) > parse_version(version): plan.notes.append(("INFO", f"FastSurfer {newest} is available ({prefix}-v{newest}); --update " f"switches to it.")) + elif newest_of_several: + tag = candidates[-1] + plan = place_release(tag, checked=True) + plan.notes.append(("INFO", f"Several FastSurfer images are installed, this uses the newest, {tag}; " + f"choose another with --image .")) else: def stores(t: str) -> str: return " and ".join(s for s, has in (("Docker", t in docker_images), ("Apptainer", t in sif_images)) if has) @@ -1232,7 +1245,7 @@ def info(message: str, level: str = "INFO") -> None: plan = decide_image(kind, value, build.prefix, options.runtime, installed, local_docker_images() if "docker" in installed else None, local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, - used) + used, newest_of_several=options.exec is not None) runtime = plan.runtime if hub is not None: stable = stable_launcher() From f341e67c172d872991f67d7255408c49fa2f1cd7 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Fri, 16 Oct 2026 18:38:19 +0200 Subject: [PATCH 06/15] Drop the advice on keeping the image for --exec --- test/utils/test_fastsurfer_container.py | 5 +++-- tools/fastsurfer_container.py | 17 +++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 32d9f35ae..461adba25 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -299,10 +299,11 @@ def test_decide_image_several_installed(tmp_path): sif={"cu132-v2.6.0": ""}).tag == "cu132-v2.6.0" assert decide(tmp_path, runtime="docker", docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}).tag == "cu132-v2.5.4" - # a program other than FastSurfer (--exec) takes the newest instead of stopping + # a program other than FastSurfer (--exec) takes the newest instead of stopping, without study advice plan = launcher.decide_image("auto", "", "cu132", "auto", BOTH, {"cu132-v2.5.4": "sha256:b"}, {"cu132-v2.6.0": ""}, - HUB, tmp_path, False, newest_of_several=True) + HUB, tmp_path, False, other_program=True) assert plan.tag == "cu132-v2.6.0" and any("uses the newest" in note for _, note in plan.notes) + assert not any("For a study" in note for _, note in plan.notes) def test_decide_image_digest(tmp_path): diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index b17c46d5f..2d86d42a5 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -591,7 +591,7 @@ def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: se docker_images: dict[str, str] | None, sif_images: dict[str, str], hub: dict[str, set[str]] | None, image_dir: Path, update: bool, local_id: str | None = None, gpus: list[Gpu] | None = None, - newest_of_several: bool = False) -> ImagePlan: + other_program: bool = False) -> ImagePlan: """ Which image to run, with which runtime, and whether it has to be downloaded or built first. @@ -626,9 +626,9 @@ def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: se gpus : list[Gpu], None The GPUs the run will use, to tell which installed images run here; empty for a run on the CPU. None counts only images of type `prefix`. - newest_of_several : bool, default=False - Use the newest of several installed images instead of stopping, for a program other than - FastSurfer, whose results do not depend on the image. + other_program : bool, default=False + For a program other than FastSurfer (--exec), whose results do not depend on the image: use the + newest of several installed images instead of stopping, and give no advice on keeping the image. """ apptainer = _apptainer_runtime(runtime, installed) docker_images = docker_images or {} @@ -733,7 +733,7 @@ def runs_here(image_type: str) -> bool: if newest is not None and parse_version(newest) > parse_version(version): plan.notes.append(("INFO", f"FastSurfer {newest} is available ({prefix}-v{newest}); --update " f"switches to it.")) - elif newest_of_several: + elif other_program: tag = candidates[-1] plan = place_release(tag, checked=True) plan.notes.append(("INFO", f"Several FastSurfer images are installed, this uses the newest, {tag}; " @@ -752,8 +752,9 @@ def stores(t: str) -> str: hint = f"e.g. --image {candidates[-1]}" raise LauncherError(f"Several FastSurfer images that run here are installed: {'; '.join(where)}. Choose " f"one with --image , {hint}.") - plan.notes.insert(0, ("INFO", f"Running FastSurfer {plan.tag.rsplit('-v', 1)[-1]} ({plan.tag}). For a study, " - f"keep this image with --image {plan.tag}.")) + if not other_program: + plan.notes.insert(0, ("INFO", f"Running FastSurfer {plan.tag.rsplit('-v', 1)[-1]} ({plan.tag}). For a " + f"study, keep this image with --image {plan.tag}.")) return plan @@ -1245,7 +1246,7 @@ def info(message: str, level: str = "INFO") -> None: plan = decide_image(kind, value, build.prefix, options.runtime, installed, local_docker_images() if "docker" in installed else None, local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, - used, newest_of_several=options.exec is not None) + used, other_program=options.exec is not None) runtime = plan.runtime if hub is not None: stable = stable_launcher() From f2d46597663cff82ef9ad326061b0e9966cc7e12 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 00:31:10 +0200 Subject: [PATCH 07/15] Run the ROCm image on AMD GPUs in the launcher (experimental) --- README.md | 2 +- doc/overview/CONTAINERS.md | 2 + doc/overview/QUICKSTART.md | 3 +- doc/overview/install/LINUX.md | 11 +- test/lint/test_launcher_builds.py | 7 +- test/utils/test_fastsurfer_container.py | 87 ++++++++++ tools/fastsurfer_container.py | 221 +++++++++++++++++++++--- 7 files changed, 304 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 407cc883e..f68f10319 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ The images we provide on [Docker Hub](https://hub.docker.com/r/deepmi/fastsurfer ### Usage All installation methods use the `run_fastsurfer.sh` call interface (replace the placeholder `` with [FastSurfer flags](doc/scripts/RUN_FASTSURFER.md#required-arguments)), which is the general starting point for FastSurfer. However, there are different ways to call this script depending on the installation, which we explain here: -1. For container installations, the [launcher](doc/overview/CONTAINERS.md#the-launcher) is the easiest way: it takes the `` and writes and runs the container command, with the image for your NVIDIA GPU (or the CPU): +1. For container installations, the [launcher](doc/overview/CONTAINERS.md#the-launcher) is the easiest way: it takes the `` and writes and runs the container command, with the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU): ```bash curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index ad5bd4b5c..2e6ce2976 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -28,6 +28,8 @@ The launcher - picks the image that runs on your NVIDIA GPU and driver, or the CPU image without one, see [which image fits your GPU](install/LINUX.md#nvidia-gpus); with several GPUs and no `--device cuda:`, it uses the newest one with enough memory and says which, +- runs the [ROCm image](install/LINUX.md#amd-gpus-experimental) on an AMD GPU (experimental) if there + is no NVIDIA GPU, or with `--image rocm`, - downloads the image once (for Apptainer, into `$HOME/my_singularity_images`), - shares the files and directories the flags name with the container, at the same paths, so relative paths work too, diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index d5c31cd42..6f6db9a28 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -21,7 +21,8 @@ This needs a working installation of Apptainer, Singularity or Docker, which the [Intel Macs](install/MACOS.md#docker-intel-macs). `docker run --rm hello-world` or `apptainer --version` checks it. The [launcher](CONTAINERS.md#the-launcher) then writes the container command for you: it picks the image for your -NVIDIA GPU (or the CPU), downloads it on the first run, and shares this directory with the container. +NVIDIA GPU (or AMD GPU, experimental, or the CPU), downloads it on the first run, and shares this directory with the +container. ```bash # 0. Create a directory for this test diff --git a/doc/overview/install/LINUX.md b/doc/overview/install/LINUX.md index 8498ae3eb..9370d6f02 100644 --- a/doc/overview/install/LINUX.md +++ b/doc/overview/install/LINUX.md @@ -10,8 +10,8 @@ A GPU with enough memory makes the segmentation much faster, see the [system requirements](../intro.rst#system-requirements). Without a GPU, use the [CPU image](#cpu-only). With Apptainer, Singularity or Docker installed, the [launcher](../CONTAINERS.md#the-launcher) does the rest: it -picks the image for your NVIDIA GPU and driver (or the CPU image), downloads it, and runs FastSurfer with the flags of -`run_fastsurfer.sh`: +picks the image for your NVIDIA GPU and driver (or for an AMD GPU, experimental, or the CPU image), downloads it, +and runs FastSurfer with the flags of `run_fastsurfer.sh`: ```bash curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py @@ -121,7 +121,12 @@ semi-supported) GPU and the right kernel version. Install the AMD kernel modules [ROCm installation instructions](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/), and add your user to the groups they name. -Then download our image for ROCm {{ ROCM_VERSION }}: +The [launcher](../CONTAINERS.md#the-launcher) uses the ROCm image when the machine has an AMD GPU and no NVIDIA GPU, +or when you pass `--image rocm`. It runs with Docker or Apptainer and adds the flags below. Its image supports +several AMD architectures; for another GPU of the same family, such as the W6600 (gfx1032, family gfx1030), it sets +`HSA_OVERRIDE_GFX_VERSION` and says so. + +Without the launcher, download our image for ROCm {{ ROCM_VERSION }}: ```bash docker pull deepmi/fastsurfer:{{ ROCM_STRING }}-v{{ FASTSURFER_VERSION }} diff --git a/test/lint/test_launcher_builds.py b/test/lint/test_launcher_builds.py index df44de20c..73abb4e08 100644 --- a/test/lint/test_launcher_builds.py +++ b/test/lint/test_launcher_builds.py @@ -55,5 +55,8 @@ def test_launcher_capabilities_cover_builds(): def test_launcher_default_build_matches_pyproject(): with open(FASTSURFER_HOME / "pyproject.toml", "rb") as fp: - cuda = tomllib.load(fp)["tool"]["cuda"]["version"] - assert launcher.NEWEST_BUILD[0] == tuple(int(v) for v in cuda.split(".")) + tool = tomllib.load(fp)["tool"] + assert launcher.NEWEST_BUILD[0] == tuple(int(v) for v in tool["cuda"]["version"].split(".")) + # the ROCm image; a new version also needs its architectures (ROCM_ARCHS) read from its PyTorch + rocm = tool["rocm"]["version"] + assert launcher.ROCM_BUILD == (tuple(int(v) for v in rocm.split(".")), f"rocm{rocm}") diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 461adba25..e5400dd57 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -485,6 +485,93 @@ def test_build_image(tmp_path): assert sorted(p.name for p in sif.parent.iterdir()) == sorted([sif.name, failed.name]) +def amd_topology(tmp_path, nodes): + """A stand-in for the kernel driver's /dev/kfd and topology: one node per (gfx_target_version, GiB).""" + device, topology = tmp_path / "kfd", tmp_path / "nodes" + device.write_text("") + for number, (target, gib) in enumerate(nodes): + node = topology / str(number) + (node / "mem_banks" / "0").mkdir(parents=True) + (node / "properties").write_text(f"cpu_cores_count 0\ngfx_target_version {target}\n") + (node / "mem_banks" / "0" / "properties").write_text(f"heap_type 1\nsize_in_bytes {gib * 2**30}\n") + return device, topology + + +def test_query_amd_gpus(tmp_path): + # node 0 is the CPU, without a GPU architecture; the GPUs are numbered from 0 as ROCm does + device, topology = amd_topology(tmp_path, [(0, 64), (100302, 8), (110000, 24)]) + gpus = launcher.query_amd_gpus(device, topology) + assert [(g.index, g.arch, g.memory_gib) for g in gpus] == [(0, "gfx1032", 8), (1, "gfx1100", 24)] + # without the kernel driver, no AMD GPU + assert launcher.query_amd_gpus(tmp_path / "missing", topology) == [] + + +@pytest.mark.parametrize( + ("version", "override"), + [ + ((11, 0, 0), ""), # gfx1100, in the image + ((9, 0, 10), ""), # gfx90a, the stepping is hexadecimal + ((10, 3, 2), "10.3.0"), # gfx1032 (Radeon Pro W6600), runs as gfx1030 + ((9, 0, 12), "9.0.0"), # gfx90c, an APU, runs as gfx900 + ((8, 0, 3), None), # gfx803, no image of its family + ], +) +def test_amd_override(version, override): + assert launcher.amd_override(launcher.AmdGpu(0, version)) == override + + +def test_plan_amd_build(): + w6600, rx7900 = launcher.AmdGpu(0, (10, 3, 2), 8), launcher.AmdGpu(1, (11, 0, 0), 24) + # the GPU with the most memory, shown alone to the container, without an override + build = launcher.plan_amd_build([], [w6600, rx7900], False) + assert (build.prefix, build.gpu, build.amd, build.devices, build.env) == ("rocm7.2", True, True, [1], []) + assert "experimental" in build.notes[0] + # a GPU of a supported family runs with the override, which says so + alone = launcher.plan_amd_build([], [w6600], False) + assert alone.env == ["HSA_OVERRIDE_GFX_VERSION=10.3.0"] and "gfx1030" in alone.notes[1] + # with --brun, only GPUs that run with the same override, as it applies to the whole container + assert launcher.plan_amd_build([], [w6600, rx7900], True).devices == [1] + # named GPUs as named; one that does not exist stops + assert launcher.plan_amd_build(["--device", "cuda:0"], [w6600, rx7900], False).devices is None + with pytest.raises(launcher.LauncherError, match="no AMD GPU 2"): + launcher.plan_amd_build(["--device", "cuda:2"], [w6600, rx7900], False) + # no supported GPU: the CPU, or an error if a GPU was requested + old = launcher.AmdGpu(0, (8, 0, 3), 4) + assert not launcher.plan_amd_build([], [old], False).gpu + with pytest.raises(launcher.LauncherError): + launcher.plan_amd_build(["--device", "cuda"], [old], False) + + +def test_decide_image_rocm(tmp_path): + # for an AMD GPU only the ROCm image counts among the installed ones, else the newest ROCm release + hub = {**HUB, "rocm7.2-v2.6.0": {"sha256:r"}} + installed = {"rocm7.2-v2.6.0": "sha256:r", "cu132-v2.6.0": "sha256:a"} + plan = launcher.decide_image("auto", "", "rocm7.2", "docker", BOTH, installed, {}, hub, tmp_path, False) + assert plan.image == "deepmi/fastsurfer:rocm7.2-v2.6.0" and not plan.pull + plan = launcher.decide_image("auto", "", "rocm7.2", "docker", BOTH, {"cu132-v2.6.0": "sha256:a"}, {}, hub, + tmp_path, False) + assert plan.image == "deepmi/fastsurfer:rocm7.2-v2.6.0" + + +def test_parse_image_spec_rocm(): + assert launcher.parse_image_spec("rocm") == ("rocm", "") + assert launcher.parse_image_spec("rocm7.2-v2.6.0") == ("tag", "rocm7.2-v2.6.0") + + +def test_amd_commands(tmp_path): + env = ["HSA_OVERRIDE_GFX_VERSION=10.3.0"] + docker = launcher.docker_command("img:tag", True, [], launcher.RUN_SCRIPT, [], [], "1000:1000", devices=[1], + amd=True, env=env, groups=[44, 109]) + assert "--gpus" not in docker + start = docker.index("--device") + assert docker[start:start + 15] == ["--device", "/dev/kfd", "--device", "/dev/dri", "--ipc=host", "--shm-size", + "8G", "--group-add", "44", "--group-add", "109", "--env", env[0], "--env", + "HIP_VISIBLE_DEVICES=1"] + apptainer = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], launcher.RUN_SCRIPT, [], [], + devices=[1], amd=True, env=env) + assert apptainer[2:7] == ["--rocm", "--env", env[0], "--env", "HIP_VISIBLE_DEVICES=1"] + + def test_commands(tmp_path): args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] run = launcher.RUN_SCRIPT diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 2d86d42a5..8fd997a74 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -26,8 +26,8 @@ directory. It runs the FastSurfer release image installed on the machine (the newest release if there is none, -or the image --image names), picks the image type that fits the NVIDIA GPU and driver (or the CPU -image), downloads it once, +or the image --image names), picks the image type that fits the NVIDIA GPU and driver (or, without +one, the experimental ROCm image for an AMD GPU, or the CPU image), downloads it once, shares every file and directory the flags name with the container at the same path, passes the FreeSurfer license from $FS_LICENSE if --fs_license is not given, and runs the container as the calling user. @@ -70,6 +70,15 @@ # the graphics memory FastSurfer needs at 0.8 mm, see the system requirements in the README MIN_GPU_MEMORY_GIB = 8 +# The ROCm images for AMD GPUs (experimental): (ROCm version as in tool.rocm of pyproject.toml, tag +# prefix), and the GPU architectures their PyTorch is built for, from torch.cuda.get_arch_list() in it +ROCM_BUILD = ((7, 2), "rocm7.2") +ROCM_ARCHS = {"gfx900", "gfx906", "gfx908", "gfx90a", "gfx942", "gfx950", "gfx1030", "gfx1100", "gfx1101", + "gfx1102", "gfx1103", "gfx1150", "gfx1151", "gfx1200", "gfx1201"} +# where AMD's kernel driver lists the GPUs, which the containers need, not ROCm on the host +KFD_DEVICE = Path("/dev/kfd") +KFD_TOPOLOGY = Path("/sys/class/kfd/kfd/topology/nodes") + # flags of run_fastsurfer.sh and brun_fastsurfer.sh whose value is a file that has to exist INPUT_FLAGS = {"--t1", "--t2", "--lesion_mask", "--fs_license", "--subjects_list", "--subject_list"} # flags of long_fastsurfer.sh followed by several files that have to exist, one per time point @@ -100,14 +109,35 @@ class Gpu: memory_gib: float | None = None +@dataclass +class AmdGpu: + """An AMD GPU as the kernel driver lists it, numbered as ROCm numbers the GPUs.""" + + index: int + # the architecture as (major, minor, stepping), e.g. (10, 3, 2) for gfx1032 + version: tuple[int, int, int] + memory_gib: float | None = None + + @property + def arch(self) -> str: + major, minor, stepping = self.version + return f"gfx{major}{minor:x}{stepping:x}" + + @dataclass class Build: - """The image build to run, whether the container gets GPUs, and which (None for all).""" + """ + The image build to run, whether the container gets GPUs, and which (None for all). + + `amd` marks the GPUs as AMD GPUs, and `env` holds the variables the container needs for them. + """ prefix: str gpu: bool notes: list[str] = field(default_factory=list) devices: list[int] | None = None + amd: bool = False + env: list[str] = field(default_factory=list) def parse_version(version: str) -> tuple[int, ...] | None: @@ -428,6 +458,105 @@ def plan_build_for_type(args: list[str], gpus: list[Gpu], brun: bool, image_type devices=[gpu.index for gpu in chosen]) +def _properties(path: Path) -> dict[str, str]: + """A properties file of the kernel driver's topology, lines of " ".""" + return dict(line.split()[:2] for line in path.read_text().splitlines() if len(line.split()) >= 2) + + +def query_amd_gpus(device: Path = KFD_DEVICE, topology: Path = KFD_TOPOLOGY) -> list[AmdGpu]: + """ + The AMD GPUs, from the kernel driver's topology, in the order ROCm numbers them; empty without the + driver. The topology also lists the CPUs, as nodes without a GPU architecture. + """ + if not device.exists() or not topology.is_dir(): + return [] + gpus = [] + for node in sorted((n for n in topology.iterdir() if n.name.isdigit()), key=lambda n: int(n.name)): + try: + target = int(_properties(node / "properties").get("gfx_target_version", "0")) + banks = sorted((node / "mem_banks").iterdir()) if (node / "mem_banks").is_dir() else [] + sizes = [int(_properties(bank / "properties").get("size_in_bytes", "0")) for bank in banks] + except (OSError, ValueError): + continue + if target: + # e.g. 100302 for gfx1032 + version = (target // 10000, target // 100 % 100, target % 100) + gpus.append(AmdGpu(len(gpus), version, sum(sizes) / 2**30 if sizes else None)) + return gpus + + +def amd_override(gpu: AmdGpu) -> str | None: + """ + The HSA_OVERRIDE_GFX_VERSION that makes the ROCm image run on this GPU: "" if it needs none, the version + of the first architecture of its family if the image supports that one (e.g. 10.3.0 for gfx1032), and + None if the image does not support the GPU. + """ + if gpu.arch in ROCM_ARCHS: + return "" + major, minor, _ = gpu.version + if AmdGpu(gpu.index, (major, minor, 0)).arch in ROCM_ARCHS: + return f"{major}.{minor}.0" + return None + + +def describe_amd(gpu: AmdGpu) -> str: + """E.g. "GPU 0 (gfx1100, 24 GB)".""" + memory = f", {gpu.memory_gib:.0f} GB" if gpu.memory_gib is not None else "" + return f"GPU {gpu.index} ({gpu.arch}{memory})" + + +def plan_amd_build(args: list[str], gpus: list[AmdGpu], brun: bool, image_type: str = ROCM_BUILD[1]) -> Build: + """ + The ROCm image and the AMD GPUs for these arguments (experimental). + + As for NVIDIA GPUs, named GPUs are used as named, and otherwise the one with the most memory; with + brun_fastsurfer.sh, every GPU that runs with the same HSA_OVERRIDE_GFX_VERSION, as it applies to all. + """ + use, named = gpu_request(args, brun) + if use == "cpu": + return Build(image_type, False) + if named is not None: + by_index = {gpu.index: gpu for gpu in gpus} + missing = [i for i in named if i not in by_index] + if missing: + raise LauncherError(f"There is no AMD GPU {missing[0]}, the driver lists {len(gpus)}.") + chosen = [by_index[i] for i in named] + unsupported = [gpu for gpu in chosen if amd_override(gpu) is None] + if unsupported: + raise LauncherError(f"The ROCm image does not support the AMD {describe_amd(unsupported[0])}.") + if len({amd_override(gpu) for gpu in chosen}) > 1: + raise LauncherError("The named AMD GPUs are of different kinds, which cannot run together; name GPUs " + "of one kind.") + devices = None + else: + supported = [gpu for gpu in gpus if amd_override(gpu) is not None] + if not supported: + reason = ("No AMD GPU found" if not gpus else + f"The ROCm image supports none of the AMD GPUs here ({', '.join(map(describe_amd, gpus))})") + if use == "cuda": + raise LauncherError(f"{reason}.") + return Build(image_type, False, [f"{reason}, so FastSurfer runs on the CPU."]) + best = max(supported, key=lambda gpu: ((gpu.memory_gib or 0) >= MIN_GPU_MEMORY_GIB, gpu.memory_gib or 0)) + chosen = [gpu for gpu in supported if amd_override(gpu) == amd_override(best)] if brun else [best] + devices = [gpu.index for gpu in chosen] + override = amd_override(chosen[0]) + notes = [f"Using the AMD {', '.join(map(describe_amd, chosen))} with the ROCm image, which is experimental; " + f"--device cpu runs on the CPU instead."] + env = [] + if override: + env.append(f"HSA_OVERRIDE_GFX_VERSION={override}") + family = AmdGpu(0, chosen[0].version[:2] + (0,)).arch + notes.append(f"The ROCm image is not built for {chosen[0].arch}, so it runs the GPU as {family} " + f"(HSA_OVERRIDE_GFX_VERSION={override}), which works for many GPUs of this family.") + return Build(image_type, True, notes, devices, amd=True, env=env) + + +def amd_device_groups(device: Path = KFD_DEVICE, dri: Path = Path("/dev/dri")) -> list[int]: + """The groups that own the devices of AMD GPUs, which a Docker container running as the user needs.""" + paths = [device] + (sorted(dri.glob("renderD*")) if dri.is_dir() else []) + return sorted({path.stat().st_gid for path in paths if path.exists()}) + + def image_reference(prefix: str, version: str) -> str: """The Docker image to run, e.g. deepmi/fastsurfer:cu132-v2.6.0.""" return f"{DOCKER_REPO}:{prefix}-v{version}" @@ -544,12 +673,15 @@ def parse_image_spec(spec: str | None) -> tuple[str, str]: Returns ------- tuple[str, str] - ("auto", "") without --image, ("version", "2.6.0"), ("tag", "cu132-v2.6.0") for a release tag, - ("release", "cu132-v2.6.0") for the full reference deepmi/fastsurfer:cu132-v2.6.0, ("sif", path) - for an Apptainer file and ("docker", reference) for any other Docker image. + ("auto", "") without --image, ("rocm", "") for the newest ROCm image, ("version", "2.6.0"), + ("tag", "cu132-v2.6.0") for a release tag, ("release", "cu132-v2.6.0") for the full reference + deepmi/fastsurfer:cu132-v2.6.0, ("sif", path) for an Apptainer file and ("docker", reference) for + any other Docker image. """ if not spec: return "auto", "" + if spec == "rocm": + return "rocm", "" if spec.endswith(".sif"): return "sif", spec if parse_version(spec): @@ -935,18 +1067,25 @@ def runtime_too_old(version_text: str) -> str | None: CUDA_ORDER = "CUDA_DEVICE_ORDER=PCI_BUS_ID" +def _gpu_env(amd: bool, devices: list[int] | None, env: list[str] | None) -> list[str]: + """The environment variables that select the GPUs in the container, for NVIDIA or AMD GPUs.""" + variables = list(env or []) if amd else [CUDA_ORDER] + if devices is not None: + variables.append(f"{'HIP' if amd else 'CUDA'}_VISIBLE_DEVICES={','.join(map(str, devices))}") + return [arg for variable in variables for arg in ("--env", variable)] + + def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], program: str, args: list[str], extra: list[str], devices: list[int] | None = None, - workdir: Path | None = None) -> list[str]: + workdir: Path | None = None, amd: bool = False, env: list[str] | None = None) -> list[str]: """ The Apptainer/Singularity command running `program`, showing only `devices` if given and starting in - `workdir` if given; the home directory is not mounted. + `workdir` if given; the home directory is not mounted. `amd` gives it AMD GPUs instead of NVIDIA ones, + with the variables in `env`. """ cmd = [runtime, "exec"] if gpu: - cmd += ["--nv", "--env", CUDA_ORDER] - if devices is not None: - cmd += ["--env", f"CUDA_VISIBLE_DEVICES={','.join(map(str, devices))}"] + cmd += ["--rocm" if amd else "--nv"] + _gpu_env(amd, devices, env) cmd += ["--no-mount", "home,cwd", "-e"] for mount in mounts: cmd += ["-B", str(mount)] @@ -957,22 +1096,31 @@ def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], pr def docker_command(image: str, gpu: bool, mounts: list[Path], program: str, args: list[str], extra: list[str], user: str, emulate: bool = False, devices: list[int] | None = None, - workdir: Path | None = None, interactive: bool = False) -> list[str]: + workdir: Path | None = None, interactive: bool = False, amd: bool = False, + env: list[str] | None = None, groups: list[int] | None = None) -> list[str]: """ The Docker command running `program`, as the calling user, showing only `devices` if given and starting in `workdir` if given. `emulate` runs the Intel image on Apple silicon; `interactive` connects the terminal, e.g. for a shell. + `amd` gives it AMD GPUs instead of NVIDIA ones, with the variables in `env` and the user in `groups`, + the groups that own the GPU devices. """ # --init: the image's entrypoint is a shell, which as the first process ignores Ctrl-C and the # signal a cancelled cluster job sends; tini passes them to all of FastSurfer's processes instead cmd = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1", "--user", user] if interactive: cmd += ["-it"] - if gpu: + if gpu and amd: + # the devices of the kernel driver, and the shared memory settings of AMD's documented PyTorch command + cmd += ["--device", "/dev/kfd", "--device", "/dev/dri", "--ipc=host", "--shm-size", "8G"] + for group in groups or []: + cmd += ["--group-add", str(group)] + cmd += _gpu_env(True, devices, env) + elif gpu: # the quotes are part of the value: without them, Docker splits a list of devices at the comma selection = "all" if devices is None else f'"device={",".join(map(str, devices))}"' - cmd += ["--gpus", selection, "--env", CUDA_ORDER] + cmd += ["--gpus", selection] + _gpu_env(False, None, None) if emulate: cmd += ["--platform", "linux/amd64"] if platform.system() == "Linux" and Path("/etc/localtime").exists(): @@ -1121,9 +1269,11 @@ def make_parser() -> argparse.ArgumentParser: ) parser.add_argument("--image", help="the image to run: a release tag such as cu132-v2.6.0 (to keep a study on one " - "image), a version such as 2.6.0 (the image type is chosen for this machine), a " - "Docker image (e.g. my_fastsurfer:cu132) or an Apptainer .sif file. Default: the " - "release image installed on this machine, or the newest release if there is none") + "image), a version such as 2.6.0 (the image type is chosen for this machine), rocm " + "for the ROCm image on an AMD GPU (experimental; used without --image when there is " + "no NVIDIA GPU), a Docker image (e.g. my_fastsurfer:cu132) or an Apptainer .sif file. " + "Default: the release image installed on this machine, or the newest release if " + "there is none") parser.add_argument("--update", action="store_true", help="download the newest release (or the one --image names) again, even if an image " "is installed, and run it") @@ -1221,16 +1371,39 @@ def info(message: str, level: str = "INFO") -> None: # the GPUs and the image type; another program's flags say nothing about them kind, value = parse_image_spec(options.image) gpus = query_nvidia_gpus() + amd_gpus = query_amd_gpus() request = [] if options.exec is not None else args # whether the newest images support none of the GPUs, so they run on the CPU here gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None wants_gpu = gpu_request(request, options.brun)[0] != "cpu" and bool(gpus) - if kind in ("auto", "version"): + image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None + # AMD GPUs (experimental): when named with --image, or when there is no NVIDIA GPU + amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) + rocm = (kind == "rocm" or (image_type or "").startswith("rocm") + or (kind in ("sif", "docker") and "rocm" in value.lower())) + if rocm and not amd_gpus: + raise LauncherError(f"The ROCm images are for AMD GPUs, and AMD's kernel driver lists none here " + f"({KFD_DEVICE} is missing), see {DOC_URL}.") + if rocm: + build = plan_amd_build(request, amd_gpus, options.brun, image_type or ROCM_BUILD[1]) + if kind in ("sif", "docker"): + build.prefix = "custom" + elif kind in ("auto", "version") and not gpus and amd_runs and gpu_request(request)[0] != "cpu": + build = plan_amd_build(request, amd_gpus, options.brun) + elif kind in ("auto", "version"): build = plan_build(request, gpus, options.brun) + if gpus and amd_runs: + build.notes.append("This machine also has an AMD GPU; --image rocm runs FastSurfer on it instead " + "(experimental).") + elif amd_gpus and not amd_runs: + build.notes.append(f"The ROCm image supports none of the AMD GPUs here " + f"({', '.join(map(describe_amd, amd_gpus))}).") elif kind in ("tag", "release"): - build = plan_build_for_type(request, gpus, options.brun, RELEASE_TAG.fullmatch(value).group(1)) + build = plan_build_for_type(request, gpus, options.brun, image_type) else: build = Build("custom", wants_gpu) + if kind == "rocm": + kind = "auto" # the image: one request to Docker Hub decides about versions hub = None if kind in ("sif", "docker") else hub_tags() @@ -1241,8 +1414,11 @@ def info(message: str, level: str = "INFO") -> None: f"launcher does not know the images of the newest release. Update it with " f"--update_launcher, or download it again from {LAUNCHER_URL}.") local_id = local_docker_id(value) if kind == "docker" else None - # the GPUs this run uses, to tell which installed images run here; none for a run on the CPU + # the GPUs this run uses, to tell which installed images run here; none for a run on the CPU, and + # for AMD GPUs only the ROCm image of this launcher counts used = [g for g in gpus if build.devices is None or g.index in build.devices] if build.gpu else [] + if build.amd or build.prefix.startswith("rocm"): + used = None plan = decide_image(kind, value, build.prefix, options.runtime, installed, local_docker_images() if "docker" in installed else None, local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, @@ -1290,7 +1466,8 @@ def info(message: str, level: str = "INFO") -> None: # Apptainer passes the terminal on itself; Docker only when asked interactive = options.exec is not None and sys.stdin.isatty() and sys.stdout.isatty() commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, - build.devices, workdir, interactive)) + build.devices, workdir, interactive, build.amd, build.env, + amd_device_groups() if build.amd else None)) if not options.print_only: memory = docker_memory_gib() if memory < MIN_DOCKER_MEMORY_GIB: @@ -1304,7 +1481,7 @@ def info(message: str, level: str = "INFO") -> None: if plan.build_from is not None: commands.append([runtime, "build", plan.image, plan.build_from]) commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, program, args, extra, - build.devices, workdir)) + build.devices, workdir, build.amd, build.env)) except LauncherError as error: print(f"ERROR: {error}", file=sys.stderr) return 1 From 5e0146e91f02c7f075501949f13dc90c16afa2cc Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 00:46:39 +0200 Subject: [PATCH 08/15] Find the AMD render nodes without a name codespell reads as a typo --- tools/fastsurfer_container.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 8fd997a74..013885948 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -553,7 +553,8 @@ def plan_amd_build(args: list[str], gpus: list[AmdGpu], brun: bool, image_type: def amd_device_groups(device: Path = KFD_DEVICE, dri: Path = Path("/dev/dri")) -> list[int]: """The groups that own the devices of AMD GPUs, which a Docker container running as the user needs.""" - paths = [device] + (sorted(dri.glob("renderD*")) if dri.is_dir() else []) + # the render nodes of the GPUs + paths = [device] + (sorted(dri.glob("render*")) if dri.is_dir() else []) return sorted({path.stat().st_gid for path in paths if path.exists()}) From 6be48087001136104a79797bddcdda31e40424b2 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 08:30:50 +0200 Subject: [PATCH 09/15] Run brun_fastsurfer.sh and long_fastsurfer.sh with --exec, accept --exec=PROGRAM, and pass piped input to --exec programs --- doc/overview/CONTAINERS.md | 24 +++-- test/utils/test_fastsurfer_container.py | 23 ++++- tools/fastsurfer_container.py | 114 +++++++++++++----------- 3 files changed, 100 insertions(+), 61 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 2e6ce2976..5a4b2e51e 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -61,20 +61,28 @@ version, it says how many releases it is behind. A GPU that the newest images no them on the CPU. The image of an older release may still support that GPU, but the launcher recommends the newest release on the CPU, or a machine with a newer GPU, over an old version. -With `--brun`, it runs `brun_fastsurfer.sh` for several subjects; the paths in a subjects list have -to be absolute. With `--long`, it runs `long_fastsurfer.sh` for the time points of one person (see -[long_fastsurfer.sh](../scripts/long_fastsurfer.rst)). +`--exec ` runs another program of the image instead of `run_fastsurfer.sh`. It comes last, as +all arguments after the program go to the program. The other FastSurfer scripts take FastSurfer's flags +and are handled like `run_fastsurfer.sh`: -`--exec ` runs any other program of the image, for example to convert a segmentation to NIfTI: +```bash +# several subjects; the paths in a subjects list have to be absolute +python3 fastsurfer_container.py --exec brun_fastsurfer.sh --subjects_list subjects.txt --sd results +# the time points of one person (longitudinal processing) +python3 fastsurfer_container.py --exec long_fastsurfer.sh --tid person --t1s tp1.nii.gz tp2.nii.gz \ + --tpids tp1 tp2 --sd results +``` + +([long_fastsurfer.sh](../scripts/long_fastsurfer.rst) explains longitudinal processing.) Any other +program runs in the current directory, for example to convert a segmentation to NIfTI: ```bash python3 fastsurfer_container.py --exec nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz aparc.nii.gz ``` -It comes last, as all arguments after the program go to the program. The program runs in the current -directory, which the launcher shares with the container, as it does the directories of absolute paths -in the arguments. Relative paths therefore only work inside the current directory. If several FastSurfer -images are installed, `--exec` uses the newest instead of asking. +The launcher shares the current directory with the container, as it does the directories of absolute +paths in the arguments, so relative paths only work inside the current directory. If several FastSurfer +images are installed, such a program uses the newest instead of asking. `--print_only` prints the commands instead of running them, `--update_launcher` replaces the launcher with the current one, and `--help` lists all launcher options. The sections diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index e5400dd57..c30bede61 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -123,7 +123,7 @@ def test_gpu_request(): assert launcher.gpu_request(["--device", "cuda:0-2,5"], brun=True) == ("cuda", [0, 1, 2, 5]) for value in ("cuda:0,1", "cuda:0-3"): for flag in ("--device", "--viewagg_device"): - with pytest.raises(launcher.LauncherError, match="--brun"): + with pytest.raises(launcher.LauncherError, match="brun_fastsurfer.sh"): launcher.gpu_request([flag, value]) assert launcher.gpu_request(["--device", "cpu"]) == ("cpu", None) # the view aggregation can need a GPU while the segmentation runs on the CPU @@ -433,6 +433,20 @@ def test_split_exec(): # the program's own flags, even the launcher's --help and --image, reach the program assert (options.runtime, options.exec, rest) == ("docker", "mri_convert", []) assert program_args == ["--help", "--image", "x"] + # the same with --exec=, also without arguments for it + argv, program_args = launcher.split_exec(["--exec=nib-convert", "in.mgz", "out.nii.gz"]) + assert (launcher.make_parser().parse_known_args(argv)[0].exec, program_args) == ("nib-convert", + ["in.mgz", "out.nii.gz"]) + assert launcher.split_exec(["--exec=bash"]) == (["--exec=bash"], []) + + +def test_fastsurfer_script(): + # the FastSurfer scripts by name or by their path in the image; anything else is another program + assert launcher.fastsurfer_script("brun_fastsurfer.sh") == launcher.BRUN_SCRIPT + assert launcher.fastsurfer_script("/fastsurfer/long_fastsurfer.sh") == "/fastsurfer/long_fastsurfer.sh" + assert launcher.fastsurfer_script("run_fastsurfer.sh") == launcher.RUN_SCRIPT + for other in ("nib-convert", "/somewhere/else/brun_fastsurfer.sh", "recon-surf.sh"): + assert launcher.fastsurfer_script(other) is None def test_link_directories(tmp_path): @@ -592,7 +606,7 @@ def test_commands(tmp_path): emulated = launcher.docker_command("img:tag", False, [tmp_path], run, args, [], "501:20", True) assert emulated[:10] == init + ["--user", "501:20", "--platform", "linux/amd64"] # the other scripts and programs run through the image's entrypoint, which sets up the environment - for program in (launcher.SCRIPTS["brun"], launcher.SCRIPTS["long"], "nib-convert"): + for program in (launcher.BRUN_SCRIPT, launcher.SCRIPTS["long_fastsurfer.sh"], "nib-convert"): other = launcher.docker_command("img:tag", False, [tmp_path], program, args, [], "1000:1000") assert other[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", "img:tag", program] @@ -606,6 +620,9 @@ def test_commands(tmp_path): workdir=tmp_path) assert in_dir[-6:] == ["--pwd", str(tmp_path), str(sif), "nib-convert"] + exec_args docker_exec = launcher.docker_command("img:tag", False, [tmp_path], "bash", [], [], "1:1", workdir=tmp_path, - interactive=True) + tty=True) assert docker_exec[8] == "-it" assert docker_exec[docker_exec.index("-w") + 1] == str(tmp_path) + # input from a pipe or file is passed on without a terminal, which Docker refuses for such input + piped = launcher.docker_command("img:tag", False, [tmp_path], "bash", [], [], "1:1", interactive=True) + assert piped[8] == "-i" diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 013885948..19f07786f 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -16,14 +16,14 @@ """ Run FastSurfer in its container image, without writing the container command. -Takes the flags of run_fastsurfer.sh (or, with --brun, of brun_fastsurfer.sh, with --long, of -long_fastsurfer.sh) and runs them in the official image with Apptainer, Singularity or Docker, -whichever is installed: +Takes the flags of run_fastsurfer.sh and runs them in the official image with Apptainer, Singularity +or Docker, whichever is installed: python3 fastsurfer_container.py --t1 data/t1.nii.gz --sd results --sid subject1 ---exec runs any other program of the image, e.g. --exec nib-convert in.mgz out.nii.gz, in the current -directory. +--exec runs another program of the image with the arguments after it: brun_fastsurfer.sh and +long_fastsurfer.sh as FastSurfer itself, and any other program, e.g. --exec nib-convert in.mgz +out.nii.gz, in the current directory. It runs the FastSurfer release image installed on the machine (the newest release if there is none, or the image --image names), picks the image type that fits the NVIDIA GPU and driver (or, without @@ -90,9 +90,10 @@ "--aseg_segfile", "--mask_name", "--cereb_segfile", "--cereb_statsfile", "--hypo_segfile", "--hypo_statsfile", "--statusfile", "--log", } -# the FastSurfer scripts the launcher runs: the default, and those of --brun and --long +# the FastSurfer scripts in the image: run_fastsurfer.sh by default, the others with --exec RUN_SCRIPT = "/fastsurfer/run_fastsurfer.sh" -SCRIPTS = {"brun": "/fastsurfer/brun_fastsurfer.sh", "long": "/fastsurfer/long_fastsurfer.sh"} +BRUN_SCRIPT = "/fastsurfer/brun_fastsurfer.sh" +SCRIPTS = {script.rsplit("/", 1)[-1]: script for script in (RUN_SCRIPT, BRUN_SCRIPT, "/fastsurfer/long_fastsurfer.sh")} class LauncherError(Exception): @@ -395,7 +396,7 @@ def gpu_request(args: list[str], brun: bool = False) -> tuple[str, list[int] | N several = [v for v in cuda if v.startswith("cuda:") and not re.fullmatch(r"cuda:\d+", v)] if several and not brun: raise LauncherError(f"{several[0]}: run_fastsurfer.sh uses one GPU, e.g. cuda:1. Lists and ranges of " - f"GPUs are for processing several subjects with --brun.") + f"GPUs are for processing several subjects with --exec brun_fastsurfer.sh.") named = [i for v in cuda for i in (cuda_indices(v) or [])] if named and values[0] in ("auto", "cuda"): # only the view aggregation names a GPU; the segmentation then runs on the first one @@ -1098,20 +1099,20 @@ def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], pr def docker_command(image: str, gpu: bool, mounts: list[Path], program: str, args: list[str], extra: list[str], user: str, emulate: bool = False, devices: list[int] | None = None, workdir: Path | None = None, interactive: bool = False, amd: bool = False, - env: list[str] | None = None, groups: list[int] | None = None) -> list[str]: + env: list[str] | None = None, groups: list[int] | None = None, tty: bool = False) -> list[str]: """ The Docker command running `program`, as the calling user, showing only `devices` if given and starting in `workdir` if given. - `emulate` runs the Intel image on Apple silicon; `interactive` connects the terminal, e.g. for a shell. - `amd` gives it AMD GPUs instead of NVIDIA ones, with the variables in `env` and the user in `groups`, - the groups that own the GPU devices. + `emulate` runs the Intel image on Apple silicon; `interactive` passes the input on (-i), and `tty` + gives the program a terminal (-t), e.g. for a shell. `amd` gives it AMD GPUs instead of NVIDIA ones, + with the variables in `env` and the user in `groups`, the groups that own the GPU devices. """ # --init: the image's entrypoint is a shell, which as the first process ignores Ctrl-C and the # signal a cancelled cluster job sends; tini passes them to all of FastSurfer's processes instead cmd = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1", "--user", user] - if interactive: - cmd += ["-it"] + if interactive or tty: + cmd += ["-i" + ("t" if tty else "")] if gpu and amd: # the devices of the kernel driver, and the shared memory settings of AMD's documented PyTorch command cmd += ["--device", "/dev/kfd", "--device", "/dev/dri", "--ipc=host", "--shm-size", "8G"] @@ -1287,15 +1288,13 @@ def make_parser() -> argparse.ArgumentParser: default=Path(os.environ.get("FASTSURFER_IMAGE_DIR", "~/my_singularity_images")).expanduser(), help="where Apptainer/Singularity images are kept (default: $FASTSURFER_IMAGE_DIR or " "~/my_singularity_images)") - script = parser.add_mutually_exclusive_group() - script.add_argument("--brun", action="store_true", help="run brun_fastsurfer.sh to process several subjects") - script.add_argument("--long", action="store_true", - help="run long_fastsurfer.sh for the time points of one person (longitudinal processing)") - script.add_argument("--exec", metavar="PROGRAM", - help="run another program of the image instead, e.g. --exec nib-convert in.mgz out.nii.gz. " - "It comes last: all later arguments go to the program. It runs in the current " - "directory, which is shared with the container, as are the directories of absolute " - "paths in its arguments") + parser.add_argument("--exec", metavar="PROGRAM", + help="run another program of the image instead of run_fastsurfer.sh. It comes last: all " + "later arguments go to the program. brun_fastsurfer.sh (several subjects) and " + "long_fastsurfer.sh (longitudinal processing) take FastSurfer's flags, as " + "run_fastsurfer.sh does. Any other program, e.g. --exec nib-convert in.mgz out.nii.gz, " + "runs in the current directory, which is shared with the container, as are the " + "directories of absolute paths in its arguments") parser.add_argument("--runtime_args", default="", help="additional arguments for the container runtime, e.g. --runtime_args '--group-add 1001'") parser.add_argument("--print_only", action="store_true", help="print the commands instead of running them") @@ -1304,13 +1303,22 @@ def make_parser() -> argparse.ArgumentParser: def split_exec(argv: list[str]) -> tuple[list[str], list[str] | None]: """ - The launcher's arguments up to `--exec `, and the program's arguments after it (None without - --exec), so that its flags, e.g. --help, reach the program rather than the launcher. + The launcher's arguments up to `--exec ` (or `--exec=`), and the program's arguments + after it (None without --exec), so that its flags, e.g. --help, reach the program rather than the launcher. """ - if "--exec" not in argv: - return argv, None - k = argv.index("--exec") - return argv[:k + 2], argv[k + 2:] + for k, arg in enumerate(argv): + if arg == "--exec": + return argv[:k + 2], argv[k + 2:] + if arg.startswith("--exec="): + return argv[:k + 1], argv[k + 1:] + return argv, None + + +def fastsurfer_script(program: str) -> str | None: + """The path in the image of a FastSurfer script --exec names, by name or path; None for other programs.""" + name = program.rsplit("/", 1)[-1] + script = SCRIPTS.get(name) + return script if script is not None and program in (name, script) else None def main(argv: list[str] | None = None) -> int: @@ -1322,10 +1330,14 @@ def main(argv: list[str] | None = None) -> int: if options.exec is not None: if fastsurfer_args: parser.error(f"unrecognized arguments before --exec: {' '.join(fastsurfer_args)}") - fastsurfer_args = program_args + fastsurfer_args = program_args or [] elif not fastsurfer_args: parser.print_help() return 1 + # the FastSurfer script to run, None for another program, which has its own flags + script = RUN_SCRIPT if options.exec is None else fastsurfer_script(options.exec) + other = script is None + brun = script == BRUN_SCRIPT def info(message: str, level: str = "INFO") -> None: print(f"{level}: {message}", file=sys.stderr) @@ -1349,12 +1361,13 @@ def info(message: str, level: str = "INFO") -> None: raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") if options.runtime != "auto" and options.runtime not in installed: raise LauncherError(f"{options.runtime} is not installed.") - if options.brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): + if brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get - raise LauncherError("--brun needs the subjects in --subjects_list or --subjects = ...") + raise LauncherError("brun_fastsurfer.sh needs the subjects in --subjects_list or " + "--subjects = ...") cwd = Path.cwd() - if options.exec is not None: + if other: args, mounts = fastsurfer_args, exec_mounts(fastsurfer_args, cwd) else: fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) @@ -1366,17 +1379,17 @@ def info(message: str, level: str = "INFO") -> None: if Path.home() in mounts: info("This shares your home directory with the container. Settings in it may affect " "FastSurfer, so keep the data in a subdirectory.") - if options.exec is None and "--fs_license" not in args and "--seg_only" not in args: + if not other and "--fs_license" not in args and "--seg_only" not in args: info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") # the GPUs and the image type; another program's flags say nothing about them kind, value = parse_image_spec(options.image) gpus = query_nvidia_gpus() amd_gpus = query_amd_gpus() - request = [] if options.exec is not None else args + request = [] if other else args # whether the newest images support none of the GPUs, so they run on the CPU here gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None - wants_gpu = gpu_request(request, options.brun)[0] != "cpu" and bool(gpus) + wants_gpu = gpu_request(request, brun)[0] != "cpu" and bool(gpus) image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None # AMD GPUs (experimental): when named with --image, or when there is no NVIDIA GPU amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) @@ -1386,13 +1399,13 @@ def info(message: str, level: str = "INFO") -> None: raise LauncherError(f"The ROCm images are for AMD GPUs, and AMD's kernel driver lists none here " f"({KFD_DEVICE} is missing), see {DOC_URL}.") if rocm: - build = plan_amd_build(request, amd_gpus, options.brun, image_type or ROCM_BUILD[1]) + build = plan_amd_build(request, amd_gpus, brun, image_type or ROCM_BUILD[1]) if kind in ("sif", "docker"): build.prefix = "custom" - elif kind in ("auto", "version") and not gpus and amd_runs and gpu_request(request)[0] != "cpu": - build = plan_amd_build(request, amd_gpus, options.brun) + elif kind in ("auto", "version") and not gpus and amd_runs and gpu_request(request, brun)[0] != "cpu": + build = plan_amd_build(request, amd_gpus, brun) elif kind in ("auto", "version"): - build = plan_build(request, gpus, options.brun) + build = plan_build(request, gpus, brun) if gpus and amd_runs: build.notes.append("This machine also has an AMD GPU; --image rocm runs FastSurfer on it instead " "(experimental).") @@ -1400,7 +1413,7 @@ def info(message: str, level: str = "INFO") -> None: build.notes.append(f"The ROCm image supports none of the AMD GPUs here " f"({', '.join(map(describe_amd, amd_gpus))}).") elif kind in ("tag", "release"): - build = plan_build_for_type(request, gpus, options.brun, image_type) + build = plan_build_for_type(request, gpus, brun, image_type) else: build = Build("custom", wants_gpu) if kind == "rocm": @@ -1423,7 +1436,7 @@ def info(message: str, level: str = "INFO") -> None: plan = decide_image(kind, value, build.prefix, options.runtime, installed, local_docker_images() if "docker" in installed else None, local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, - used, other_program=options.exec is not None) + used, other_program=other) runtime = plan.runtime if hub is not None: stable = stable_launcher() @@ -1451,11 +1464,8 @@ def info(message: str, level: str = "INFO") -> None: for note in build.notes: info(note) - if options.exec is not None: - program, workdir = options.exec, cwd - else: - program = SCRIPTS["brun"] if options.brun else SCRIPTS["long"] if options.long else RUN_SCRIPT - workdir = None + # another program runs in the current directory; FastSurfer's paths are absolute already + program, workdir = (options.exec, cwd) if other else (script, None) extra = shlex.split(options.runtime_args) commands = [] if runtime == "docker": @@ -1464,11 +1474,15 @@ def info(message: str, level: str = "INFO") -> None: emulate = apple_silicon and (kind != "docker" or value.startswith(DOCKER_REPO)) if plan.pull: commands.append(["docker", "pull"] + (["--platform", "linux/amd64"] if emulate else []) + [plan.image]) - # Apptainer passes the terminal on itself; Docker only when asked - interactive = options.exec is not None and sys.stdin.isatty() and sys.stdout.isatty() + # Apptainer passes input and terminal on itself, Docker only when asked: another program gets the + # input when it comes from a pipe or file, and the terminal when it runs in one. Input from a + # terminal without one, e.g. a job started in the background, would stop the job. + stdin_tty = sys.stdin is not None and sys.stdin.isatty() + interactive = other and not stdin_tty + tty = other and stdin_tty and sys.stdout.isatty() commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, build.devices, workdir, interactive, build.amd, build.env, - amd_device_groups() if build.amd else None)) + amd_device_groups() if build.amd else None, tty)) if not options.print_only: memory = docker_memory_gib() if memory < MIN_DOCKER_MEMORY_GIB: From cb4feb60a2c3f191209f7bf7f3fda42b4a101489 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 08:59:20 +0200 Subject: [PATCH 10/15] Rename the launcher to fastsurfer and run the macOS package with it on Apple silicon --- README.md | 4 +- doc/conf.py | 2 +- doc/overview/CONTAINERS.md | 22 +- doc/overview/QUICKSTART.md | 87 +--- doc/overview/install/LINUX.md | 4 +- doc/overview/install/MACOS.md | 12 + pyproject.toml | 6 +- test/lint/test_launcher_builds.py | 12 +- ...ntainer.py => test_fastsurfer_launcher.py} | 64 ++- tools/{fastsurfer_container.py => fastsurfer} | 485 ++++++++++++------ 10 files changed, 441 insertions(+), 257 deletions(-) rename test/utils/{test_fastsurfer_container.py => test_fastsurfer_launcher.py} (90%) rename tools/{fastsurfer_container.py => fastsurfer} (77%) mode change 100644 => 100755 diff --git a/README.md b/README.md index f68f10319..2c81e9ada 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,8 @@ All installation methods use the `run_fastsurfer.sh` call interface (replace the 1. For container installations, the [launcher](doc/overview/CONTAINERS.md#the-launcher) is the easiest way: it takes the `` and writes and runs the container command, with the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU): ```bash - curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py - python3 fastsurfer_container.py + curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer + ./fastsurfer ``` Written by hand, the container command needs the container's setup (`` or ``) in addition to the ``: diff --git a/doc/conf.py b/doc/conf.py index f26a6b3f5..d6e821add 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -240,7 +240,7 @@ def _tool_values_gitref(ref: str | None, *keys: str) -> tuple[str, ...]: html_theme = "furo" html_static_path = ["_static"] # served next to the pages, so the launcher downloads from deep-mi.org/FastSurfer// -html_extra_path = ["../tools/fastsurfer_container.py"] +html_extra_path = ["../tools/fastsurfer"] html_js_files = ["doc-version-link.js"] # points the announcement bar at the sibling doc tree html_title = project html_show_sphinx = False diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 5a4b2e51e..c76776dbe 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -21,7 +21,7 @@ as a "simulated computer inside the computer". The launcher ------------ -`fastsurfer_container.py` writes and runs the container command for you. It takes the same flags as +The launcher `fastsurfer` writes and runs the container command for you. It takes the same flags as `run_fastsurfer.sh` and needs only Python 3.8 or newer, not FastSurfer itself. It - uses Apptainer, Singularity or Docker, whichever is installed (choose one with `--runtime`), @@ -39,8 +39,8 @@ The launcher Download it, and run FastSurfer: ```bash -curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py -python3 fastsurfer_container.py \ +curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer +./fastsurfer \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ --sid subjectX --sd $HOME/my_fastsurfer_analysis \ --3T --threads 4 @@ -67,9 +67,9 @@ and are handled like `run_fastsurfer.sh`: ```bash # several subjects; the paths in a subjects list have to be absolute -python3 fastsurfer_container.py --exec brun_fastsurfer.sh --subjects_list subjects.txt --sd results +./fastsurfer --exec brun_fastsurfer.sh --subjects_list subjects.txt --sd results # the time points of one person (longitudinal processing) -python3 fastsurfer_container.py --exec long_fastsurfer.sh --tid person --t1s tp1.nii.gz tp2.nii.gz \ +./fastsurfer --exec long_fastsurfer.sh --tid person --t1s tp1.nii.gz tp2.nii.gz \ --tpids tp1 tp2 --sd results ``` @@ -77,7 +77,7 @@ python3 fastsurfer_container.py --exec long_fastsurfer.sh --tid person --t1s tp1 program runs in the current directory, for example to convert a segmentation to NIfTI: ```bash -python3 fastsurfer_container.py --exec nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz aparc.nii.gz +./fastsurfer --exec nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz aparc.nii.gz ``` The launcher shares the current directory with the container, as it does the directories of absolute @@ -91,10 +91,12 @@ below explain the commands it runs. Before starting many runs at once, e.g. a SLURM array, run the launcher once (e.g. on one subject), so the image is downloaded and built only once rather than by every job. -On a Mac with Apple silicon, use the [macOS package](install/MACOS.md) instead: the images are built for -Intel and AMD processors, which such a Mac only emulates, much more slowly and without its GPU. The -launcher therefore stops there and points to the package, unless you pass `--runtime docker`. With -Docker, it also warns if Docker may use less memory than FastSurfer needs. +On a Mac with Apple silicon, the launcher runs the installed [macOS package](install/MACOS.md) instead, +with the same flags and `--exec`: the images are built for Intel and AMD processors, which such a Mac +only emulates, much more slowly and without its GPU. It chooses among installed package versions as +among images, and `--image 2.6.0` keeps a study on one version. Without a package, `--update` +downloads it and opens the macOS Installer. To run an image in Docker anyway, pass `--runtime docker`. +With Docker, the launcher also warns if Docker may use less memory than FastSurfer needs. Using Apptainer (or Singularity) -------------------------------- diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index 6f6db9a28..2b79340e3 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -1,9 +1,8 @@ Quick Start =========== -This example segments one brain MRI in a few minutes. Choose how you run FastSurfer: on a Mac with Apple silicon -(M1 or newer) with the [macOS package](install/MACOS.md), on Linux, Windows or a Mac with an Intel CPU in a container -(Apptainer, Singularity or Docker), or without installing anything in [Google Colab](#google-colab). The -[installation guide](INSTALL.md) helps to choose. +This example segments one brain MRI in a few minutes, with the same commands on Linux, Windows and macOS. You can +also run it without installing anything in [Google Colab](#google-colab). The [installation guide](INSTALL.md) +explains the options in more detail. Segment an example image ------------------------ @@ -11,99 +10,69 @@ With a GPU (8 GB of graphics memory) this takes well under a minute; without one a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 mm voxel sizes, no preprocessing, `.nii`, `.nii.gz` or `.mgz`), or the example image the commands download. +First, install what runs FastSurfer on your system: + `````{tab-set} ````{tab-item} Linux, Windows or Intel Mac :sync: container -This needs a working installation of Apptainer, Singularity or Docker, which these commands do not set up: see +A working installation of Apptainer, Singularity or Docker, which the commands below do not set up: see [Linux](install/LINUX.md), [Windows](install/WINDOWS.md) (Docker in WSL2, run the commands there) or [Intel Macs](install/MACOS.md#docker-intel-macs). `docker run --rm hello-world` or `apptainer --version` checks it. - -The [launcher](CONTAINERS.md#the-launcher) then writes the container command for you: it picks the image for your -NVIDIA GPU (or AMD GPU, experimental, or the CPU), downloads it on the first run, and shares this directory with the -container. - -```bash -# 0. Create a directory for this test -mkdir fastsurfer_test -cd fastsurfer_test - -# 1. Download the launcher (only the first time) -curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py - -# 2. Download an example brain MRI (if you don't have your own) -# If you have your own, copy it to this directory and adjust -# the filename after --t1 below. -curl -k \ - https://surfer.nmr.mgh.harvard.edu/pub/data/tutorial_data/buckner_data/tutorial_subjs/140/mri/orig.mgz \ - -o "./140_orig.mgz" - -# 3. Run FastSurfer (full brain segmentation only) -python3 fastsurfer_container.py \ - --t1 140_orig.mgz --sid subjectX --sd . \ - --seg_only --no_biasfield --no_cereb --no_hypothal -``` - -[Running FastSurfer in a container](CONTAINERS.md) shows the `singularity` and `docker` commands the launcher runs. +The launcher then picks the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU), downloads it on the +first run, and shares this directory with the container; [Running FastSurfer in a container](CONTAINERS.md) shows +the commands it runs. ```` ````{tab-item} Mac with Apple silicon :sync: macos -On a Mac with Apple silicon (M1 or newer), [install the macOS package](install/MACOS.md), open the FastSurfer app -from your Applications folder, and run these commands in the Terminal window it opens: +The [macOS package](install/MACOS.md) (Macs with M1 or newer), which runs natively and uses the Mac's GPU. Once +you have downloaded the launcher (step 1 below), `./fastsurfer --update` downloads the package and opens the macOS +Installer. The launcher then runs the installed package. +```` + +````` + +Then download the launcher and run FastSurfer: ```bash # 0. Create a directory for this test mkdir fastsurfer_test cd fastsurfer_test -# 1. Download an example brain MRI (if you don't have your own) +# 1. Download the launcher (only the first time) +curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer + +# 2. Download an example brain MRI (if you don't have your own) # If you have your own, copy it to this directory and adjust # the filename after --t1 below. curl -k \ https://surfer.nmr.mgh.harvard.edu/pub/data/tutorial_data/buckner_data/tutorial_subjs/140/mri/orig.mgz \ -o "./140_orig.mgz" -# 2. Run FastSurfer (full brain segmentation only) -run_fastsurfer.sh --t1 "$PWD/140_orig.mgz" \ - --sid subjectX --sd "$PWD" \ +# 3. Run FastSurfer (full brain segmentation only) +./fastsurfer --t1 140_orig.mgz --sid subjectX --sd . \ --seg_only --no_biasfield --no_cereb --no_hypothal ``` -```` -````` +The [launcher](CONTAINERS.md#the-launcher) takes the flags of `run_fastsurfer.sh`, which you can also call +directly in the Terminal window of the FastSurfer app on a Mac. That's it, it will run the full brain segmentation. For speed, we switched off the cerebellum and hypothalamic sub-segmentation (would add a couple minutes). We also switched off the bias field correction, which is used to compute partial volume estimates for the statsfiles, so you might want to switch it on again if you want the volume statistics text file (under ```subjectX/stats```). Also if you need the estimated total intracranial volume for correcting the stats, you would either need to run the surface stream or switch on the Talairach registration with -`--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license): pass it with `--fs_license `, or set the environment variable `FS_LICENSE` to its path (the launcher passes it on to the container). +`--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license): pass it with `--fs_license `, or set the environment variable `FS_LICENSE` to its path, which the launcher passes on. You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer: -`````{tab-set} - -````{tab-item} Linux, Windows or Intel Mac -:sync: container ```bash -# Convert mgz to nifti, in the image the launcher used for FastSurfer -python3 fastsurfer_container.py --exec nib-convert \ +# Convert mgz to nifti +./fastsurfer --exec nib-convert \ subjectX/mri/aparc.DKTatlas+aseg.deep.mgz \ subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz ``` -```` - -````{tab-item} Mac with Apple silicon -:sync: macos -```bash -# Convert mgz to nifti, in the Terminal window of the FastSurfer app -nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz \ - subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz -``` -```` - -````` Then find the segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz```. If you have FreeSurfer installed, just use FreeView to look at the result (or really any other image viewer): diff --git a/doc/overview/install/LINUX.md b/doc/overview/install/LINUX.md index 9370d6f02..6f09506a7 100644 --- a/doc/overview/install/LINUX.md +++ b/doc/overview/install/LINUX.md @@ -14,8 +14,8 @@ picks the image for your NVIDIA GPU and driver (or for an AMD GPU, experimental, and runs FastSurfer with the flags of `run_fastsurfer.sh`: ```bash -curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer_container.py -python3 fastsurfer_container.py --t1 --sid --sd +curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer +./fastsurfer --t1 --sid --sd ``` The sections below explain the images and how to download them by hand. diff --git a/doc/overview/install/MACOS.md b/doc/overview/install/MACOS.md index 61a384156..a66be79ab 100644 --- a/doc/overview/install/MACOS.md +++ b/doc/overview/install/MACOS.md @@ -72,6 +72,18 @@ run_fastsurfer.sh --sd $HOME/my_fastsurfer_analysis --sid subjectX \ FastSurfer uses the Apple GPU (`mps`) automatically, so no `--device` flag is needed. +Without opening the console, the [launcher](../CONTAINERS.md#the-launcher) `fastsurfer` runs the installed package +from any Terminal window, with the same flags and the same commands as on Linux: + +```bash +curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer +./fastsurfer --seg_only --sd $HOME/my_fastsurfer_analysis --sid subjectX \ + --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz +``` + +If several versions are installed, it asks for one, e.g. `--image 2.6.0`. `./fastsurfer --update` downloads the +newest package and opens the installer. + FreeSurfer license ------------------ On macOS, FastSurfer cannot find the [FreeSurfer license](../INSTALL.md#freesurfer-license) inside the bundled diff --git a/pyproject.toml b/pyproject.toml index 594aff9bd..20067df77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,9 +225,9 @@ add_ignore = 'D100,D104,D107' [tool.ruff] line-length = 120 target-version = "py312" -# recon_surf/fs_time is python, but carries no .py suffix because the pipeline invokes it as a -# command; without this ruff would skip it -extend-include = ["recon_surf/fs_time"] +# recon_surf/fs_time and the launcher tools/fastsurfer are python, but carry no .py suffix because they +# are invoked as commands; without this ruff would skip them +extend-include = ["recon_surf/fs_time", "tools/fastsurfer"] extend-exclude = [ "build", "checkpoints", diff --git a/test/lint/test_launcher_builds.py b/test/lint/test_launcher_builds.py index 73abb4e08..b4e58833c 100644 --- a/test/lint/test_launcher_builds.py +++ b/test/lint/test_launcher_builds.py @@ -13,13 +13,14 @@ # limitations under the License. """ -Guard the copy of the CUDA images in tools/fastsurfer_container.py. +Guard the copy of the CUDA images in the launcher, tools/fastsurfer. The launcher runs with any system Python and without FastSurfer installed, so it cannot import FastSurferCNN/gpu_support.py and keeps its own copy of the CUDA builds and their minimum drivers. A copy left behind when the CUDA versions change makes it pick an image that is not released. """ +import importlib.machinery import importlib.util import sys import tomllib @@ -29,15 +30,16 @@ def _load(name: str, path: Path): - # from the path, because the lint job installs nothing but pytest; registered in sys.modules, - # which dataclasses need - spec = importlib.util.spec_from_file_location(name, path) + # from the path, because the lint job installs nothing but pytest, with an explicit loader for the + # launcher, which has no .py suffix; registered in sys.modules, which dataclasses need + loader = importlib.machinery.SourceFileLoader(name, str(path)) + spec = importlib.util.spec_from_file_location(name, path, loader=loader) module = sys.modules.setdefault(name, importlib.util.module_from_spec(spec)) spec.loader.exec_module(module) return module -launcher = _load("fastsurfer_container", FASTSURFER_HOME / "tools" / "fastsurfer_container.py") +launcher = _load("fastsurfer_launcher_for_lint", FASTSURFER_HOME / "tools" / "fastsurfer") gpu_support = _load("gpu_support_for_lint", FASTSURFER_HOME / "FastSurferCNN" / "gpu_support.py") diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_launcher.py similarity index 90% rename from test/utils/test_fastsurfer_container.py rename to test/utils/test_fastsurfer_launcher.py index c30bede61..2359ad258 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_launcher.py @@ -12,19 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test the image choice and the path handling of tools/fastsurfer_container.py.""" +"""Test the image choice and the path handling of the launcher, tools/fastsurfer.""" +import importlib.machinery import importlib.util import platform +import subprocess import sys from pathlib import Path import pytest -# a script rather than a module of FastSurfer, so it is loaded from its path; dataclasses need it -# registered in sys.modules -_SCRIPT = Path(__file__).parent.parent.parent / "tools" / "fastsurfer_container.py" -_spec = importlib.util.spec_from_file_location("fastsurfer_container", _SCRIPT) +# a command rather than a module of FastSurfer, without the .py suffix, so it is loaded from its path with +# an explicit loader; dataclasses need it registered in sys.modules +_SCRIPT = str(Path(__file__).parent.parent.parent / "tools" / "fastsurfer") +_spec = importlib.util.spec_from_file_location( + "fastsurfer_launcher", _SCRIPT, loader=importlib.machinery.SourceFileLoader("fastsurfer_launcher", _SCRIPT)) launcher = sys.modules.setdefault(_spec.name, importlib.util.module_from_spec(_spec)) _spec.loader.exec_module(launcher) @@ -586,6 +589,57 @@ def test_amd_commands(tmp_path): assert apptainer[2:7] == ["--rocm", "--env", env[0], "--env", "HIP_VISIBLE_DEVICES=1"] +def macos_applications(tmp_path, names): + """A stand-in for /Applications with FastSurfer packages, each a folder with the setup script.""" + for name in names: + (tmp_path / name).mkdir() + (tmp_path / name / launcher.MACOS_SETUP).write_text("") + return tmp_path + + +def test_macos_packages(tmp_path): + # releases by version; the app that opens the console and a development build do not count + apps = macos_applications(tmp_path, ["FastSurfer2.5.4", "FastSurfer2.6.0", "FastSurfer2.7.0-dev0"]) + (tmp_path / "FastSurfer2.6.0.app").mkdir() + assert launcher.macos_packages(apps) == {"2.5.4": apps / "FastSurfer2.5.4", "2.6.0": apps / "FastSurfer2.6.0"} + # --image with the folder of a package names it directly + dev = apps / "FastSurfer2.7.0-dev0" + assert launcher.parse_image_spec(str(dev)) == ("macos", str(dev.resolve())) + + +def test_decide_macos_package(tmp_path): + packages = {"2.5.4": tmp_path / "FastSurfer2.5.4", "2.6.0": tmp_path / "FastSurfer2.6.0"} + # the same rules as for images: several installed stop the launcher, newest first + with pytest.raises(launcher.LauncherError, match="2.6.0, 2.5.4.*--image 2.6.0"): + launcher.decide_macos_package("auto", "", packages, "2.6.0") + home, notes = launcher.decide_macos_package("version", "2.5.4", packages, "2.6.0") + assert home == packages["2.5.4"] and "--image 2.5.4" in notes[0][1] + # one installed runs, with a note on a newer release + home, notes = launcher.decide_macos_package("auto", "", {"2.5.4": packages["2.5.4"]}, "2.6.0") + assert home == packages["2.5.4"] and any("2.6.0 is available" in note for _, note in notes) + # another program takes the newest, without study advice + home, notes = launcher.decide_macos_package("auto", "", packages, "2.6.0", other_program=True) + assert home == packages["2.6.0"] and not any("For a study" in note for _, note in notes) + # none installed, a version not installed, and a container image: what to do instead + with pytest.raises(launcher.LauncherError, match="--update"): + launcher.decide_macos_package("auto", "", {}, "2.6.0") + with pytest.raises(launcher.LauncherError, match="--update"): + launcher.decide_macos_package("version", "2.4.2", packages, "2.6.0") + with pytest.raises(launcher.LauncherError, match="--runtime docker"): + launcher.decide_macos_package("tag", "cu132-v2.6.0", packages, "2.6.0") + + +def test_macos_command(tmp_path): + # the package's setup script, then the program with its arguments unchanged, as bash runs it + command = launcher.macos_command(tmp_path, "nib-convert", ["a b.mgz", "out.nii.gz"]) + setup = tmp_path / launcher.MACOS_SETUP + setup.write_text("export STAND_IN=1\n") + probe = launcher.macos_command(tmp_path, sys.executable, ["-c", "import os,sys; print(os.environ['STAND_IN'], " + "sys.argv[1:])", "a b.mgz"]) + assert command[-3:] == ["nib-convert", "a b.mgz", "out.nii.gz"] + assert subprocess.run(probe, capture_output=True, text=True).stdout == "1 ['a b.mgz']\n" + + def test_commands(tmp_path): args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] run = launcher.RUN_SCRIPT diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer old mode 100644 new mode 100755 similarity index 77% rename from tools/fastsurfer_container.py rename to tools/fastsurfer index 19f07786f..7911dd08c --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer @@ -14,12 +14,13 @@ # limitations under the License. """ -Run FastSurfer in its container image, without writing the container command. +Run FastSurfer with one command on every system: in its container image, or on a Mac with Apple silicon +with the macOS package. Takes the flags of run_fastsurfer.sh and runs them in the official image with Apptainer, Singularity or Docker, whichever is installed: - python3 fastsurfer_container.py --t1 data/t1.nii.gz --sd results --sid subject1 + ./fastsurfer --t1 data/t1.nii.gz --sd results --sid subject1 --exec runs another program of the image with the arguments after it: brun_fastsurfer.sh and long_fastsurfer.sh as FastSurfer itself, and any other program, e.g. --exec nib-convert in.mgz @@ -30,10 +31,12 @@ one, the experimental ROCm image for an AMD GPU, or the CPU image), downloads it once, shares every file and directory the flags name with the container at the same path, passes the FreeSurfer license from $FS_LICENSE if --fs_license is not given, and runs the container as the -calling user. +calling user. On a Mac with Apple silicon, it runs the installed macOS package instead, with the +same choice among installed versions. Only the Python standard library is used, so this runs with the system Python of a cluster, -without FastSurfer installed. +without FastSurfer installed. It has no .py suffix, so a copy next to other Python code cannot +shadow the package fastsurfer. """ from __future__ import annotations @@ -51,11 +54,16 @@ DOCKER_REPO = "deepmi/fastsurfer" DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/CONTAINERS.html" -LAUNCHER_URL = "https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer_container.py" +LAUNCHER_URL = "https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer" # seconds to wait for Docker Hub and GitHub, so a machine without internet is not held up HUB_TIMEOUT = 5 MACOS_PACKAGE_URL = "https://github.com/Deep-MI/FastSurfer/releases/latest/download/FastSurfer-macos-darwin_arm64.pkg" MACOS_DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/install/MACOS.html" +# the macOS package installs into /Applications/FastSurfer, with a script that sets up its +# Python, FreeSurfer and PATH, as the console it opens does +MACOS_APPLICATIONS = Path("/Applications") +MACOS_SETUP = "macos_setup_fastsurfer.sh" +GITHUB_RELEASES = "https://api.github.com/repos/Deep-MI/FastSurfer/releases" # the 15 GB the macOS page asks Docker to have, less what the kernel of Docker's VM keeps for itself MIN_DOCKER_MEMORY_GIB = 14 @@ -677,13 +685,15 @@ def parse_image_spec(spec: str | None) -> tuple[str, str]: tuple[str, str] ("auto", "") without --image, ("rocm", "") for the newest ROCm image, ("version", "2.6.0"), ("tag", "cu132-v2.6.0") for a release tag, ("release", "cu132-v2.6.0") for the full reference - deepmi/fastsurfer:cu132-v2.6.0, ("sif", path) for an Apptainer file and ("docker", reference) for - any other Docker image. + deepmi/fastsurfer:cu132-v2.6.0, ("sif", path) for an Apptainer file, ("macos", folder) for a macOS + package and ("docker", reference) for any other Docker image. """ if not spec: return "auto", "" if spec == "rocm": return "rocm", "" + if (Path(spec).expanduser() / MACOS_SETUP).is_file(): + return "macos", str(Path(spec).expanduser().resolve()) if spec.endswith(".sif"): return "sif", spec if parse_version(spec): @@ -1139,6 +1149,91 @@ def docker_command(image: str, gpu: bool, mounts: list[Path], program: str, args return cmd + [image] + args +def macos_packages(applications: Path = MACOS_APPLICATIONS) -> dict[str, Path]: + """The installed macOS packages of FastSurfer releases, by version: {"2.6.0": /Applications/FastSurfer2.6.0}.""" + packages = {} + for setup in applications.glob(f"FastSurfer*/{MACOS_SETUP}"): + version = setup.parent.name[len("FastSurfer"):] + if parse_version(version) is not None: + packages[version] = setup.parent + return packages + + +def decide_macos_package(kind: str, value: str, packages: dict[str, Path], newest: str | None, + other_program: bool = False) -> tuple[Path, list[tuple[str, str]]]: + """ + The macOS package to run, and notes for the user, with the rules for container images: one installed + release runs, several stop the launcher (a program other than FastSurfer takes the newest), --image + chooses one, and --image names a package directly. + + Parameters + ---------- + kind, value : str + What --image names, see parse_image_spec. + packages : dict[str, Path] + The installed packages, see macos_packages. + newest : str, None + The newest release, None if Docker Hub cannot be reached. + other_program : bool, default=False + For a program other than FastSurfer (--exec), whose results do not depend on the version. + """ + install = "--update downloads it and opens the macOS Installer" + if kind == "macos": + return Path(value), [] + if kind not in ("auto", "version"): + raise LauncherError(f"--image {value or kind} names a container image, but on a Mac with Apple silicon " + f"FastSurfer runs from the macOS package, and --image chooses an installed one. " + f"To run the image in Docker under emulation, which is much slower, pass --runtime docker.") + notes = [] + if kind == "version": + if value not in packages: + raise LauncherError(f"FastSurfer {value} is not installed as a macOS package; {install}.") + version = value + else: + versions = sorted(packages, key=parse_version) + if not versions: + raise LauncherError(f"FastSurfer is not installed. On a Mac with Apple silicon, it runs from the macOS " + f"package: {install}, or download it from {MACOS_PACKAGE_URL} (see {MACOS_DOC_URL}).") + version = versions[-1] + if len(versions) > 1 and not other_program: + raise LauncherError(f"Several FastSurfer macOS packages are installed: {', '.join(reversed(versions))}. " + f"Choose one with --image , e.g. --image {version}, the newest installed.") + if len(versions) > 1: + notes.append(("INFO", f"Several FastSurfer macOS packages are installed, this uses the newest, {version}; " + f"choose another with --image .")) + if newest is not None and parse_version(newest) > parse_version(version): + notes.append(("INFO", f"FastSurfer {newest} is available; {install}.")) + if not other_program: + notes.insert(0, ("INFO", f"Running FastSurfer {version} (the macOS package in {packages[version]}). For a " + f"study, keep this version with --image {version}.")) + return packages[version], notes + + +def macos_package_download(version: str | None) -> tuple[str, str, str]: + """The version, download URL and file name of the macOS package of a release (the newest if None).""" + import json + + url = f"{GITHUB_RELEASES}/tags/v{version}" if version else f"{GITHUB_RELEASES}/latest" + try: + with open_url(url) as response: + release = json.load(response) + except (OSError, ValueError) as error: + raise LauncherError(f"Could not find the macOS package{f' of FastSurfer {version}' if version else ''} on " + f"GitHub: {error}") from error + # the file name differs between releases, e.g. FastSurfer254-macos-darwin_arm64.pkg + assets = [asset for asset in release.get("assets", []) + if re.fullmatch(r"FastSurfer\w*-macos-darwin_arm64\.pkg", asset.get("name", ""))] + if not assets: + raise LauncherError(f"FastSurfer {release.get('tag_name', version)} has no macOS package.") + return release["tag_name"].lstrip("v"), assets[0]["browser_download_url"], assets[0]["name"] + + +def macos_command(home: Path, program: str, args: list[str]) -> list[str]: + """The command running `program` with the macOS package in `home`, set up as in the console it opens.""" + setup = 'source "$1" > /dev/null; shift; exec "$@"' + return ["/bin/bash", "-c", setup, "bash", str(home / MACOS_SETUP), program] + args + + def is_apple_silicon() -> bool: """Whether this is a Mac with Apple silicon, also when Python itself runs under Rosetta.""" if platform.system() != "Darwin": @@ -1260,7 +1355,7 @@ def update_launcher() -> int: def make_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - prog="fastsurfer_container.py", + prog="fastsurfer", description=__doc__.split("\n\n")[0].strip(), epilog=( "All other arguments are passed to run_fastsurfer.sh (or brun_fastsurfer.sh, long_fastsurfer.sh), " @@ -1274,16 +1369,19 @@ def make_parser() -> argparse.ArgumentParser: "image), a version such as 2.6.0 (the image type is chosen for this machine), rocm " "for the ROCm image on an AMD GPU (experimental; used without --image when there is " "no NVIDIA GPU), a Docker image (e.g. my_fastsurfer:cu132) or an Apptainer .sif file. " - "Default: the release image installed on this machine, or the newest release if " + "On a Mac with Apple silicon, the version of the macOS package (e.g. 2.6.0) or its " + "folder. Default: the release installed on this machine, or the newest release if " "there is none") parser.add_argument("--update", action="store_true", help="download the newest release (or the one --image names) again, even if an image " - "is installed, and run it") + "is installed, and run it. On a Mac with Apple silicon, download the macOS package and " + "open the macOS Installer") parser.add_argument("--update_launcher", action="store_true", help="replace this launcher with the current one and stop") - parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker"], default="auto", - help="the container runtime (default: auto, the one that has the image, else Apptainer, " - "Singularity, then Docker)") + parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker", "macos"], default="auto", + help="what runs FastSurfer (default: auto, on a Mac with Apple silicon the macOS package, " + "else the container runtime that has the image, else Apptainer, Singularity, then " + "Docker)") parser.add_argument("--image_dir", type=Path, default=Path(os.environ.get("FASTSURFER_IMAGE_DIR", "~/my_singularity_images")).expanduser(), help="where Apptainer/Singularity images are kept (default: $FASTSURFER_IMAGE_DIR or " @@ -1321,6 +1419,192 @@ def fastsurfer_script(program: str) -> str | None: return script if script is not None and program in (name, script) else None +def macos_commands(options: argparse.Namespace, script: str | None, other: bool, fastsurfer_args: list[str], + info) -> list[list[str]]: + """ + The commands that run FastSurfer, or the program --exec names, with the macOS package on a Mac with + Apple silicon. With --update, they download the package and open the macOS Installer instead. + """ + kind, value = parse_image_spec(options.image) + hub = hub_tags() if kind in ("auto", "version") else None + if options.update: + version, url, name = macos_package_download(value if kind == "version" else None) + target = Path.home() / "Downloads" / name + info(f"Downloading the macOS package of FastSurfer {version} to {target}; the macOS Installer opens next. " + f"Once it is installed, run the same command again without --update.") + return [["curl", "-fL", "--progress-bar", "-o", str(target), url], ["open", str(target)]] + home, notes = decide_macos_package(kind, value, macos_packages(), newest_version(list(hub or []), "cpu"), + other) + if hub is not None: + stable = stable_launcher() + if stable is not None and stable != Path(__file__).resolve().read_bytes(): + info("A newer launcher is available, update it with --update_launcher.") + for level, note in notes: + info(note, level=level) + if other: + # it runs natively, in the current directory, so its paths need no care + return [macos_command(home, options.exec, fastsurfer_args)] + cwd = Path.cwd() + args, _ = plan_paths(fastsurfer_args + _license_args(fastsurfer_args, cwd), cwd, create=not options.print_only) + if "--fs_license" not in args and "--seg_only" not in args: + info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") + return [macos_command(home, str(home / script.rsplit("/", 1)[-1]), args)] + + +def container_commands(options: argparse.Namespace, script: str | None, other: bool, fastsurfer_args: list[str], + apple_silicon: bool, info) -> tuple[list[list[str]], str | None]: + """ + The commands that run FastSurfer, or the program --exec names, in its container image, and the digest + to record next to an Apptainer image they build (None if they build none). + """ + brun = script == BRUN_SCRIPT + installed = {name for name in ("apptainer", "singularity", "docker") if shutil.which(name)} + if not installed: + raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") + if options.runtime != "auto" and options.runtime not in installed: + raise LauncherError(f"{options.runtime} is not installed.") + if brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): + # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get + raise LauncherError("brun_fastsurfer.sh needs the subjects in --subjects_list or " + "--subjects = ...") + + cwd = Path.cwd() + if other: + args, mounts = fastsurfer_args, exec_mounts(fastsurfer_args, cwd) + else: + fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) + args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) + mounts = minimal_mounts(mounts) + if Path("/") in mounts: + raise LauncherError("The paths would share the whole file system with the container, keep the " + "data in a directory below /.") + if Path.home() in mounts: + info("This shares your home directory with the container. Settings in it may affect " + "FastSurfer, so keep the data in a subdirectory.") + if not other and "--fs_license" not in args and "--seg_only" not in args: + info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") + + # the GPUs and the image type; another program's flags say nothing about them + kind, value = parse_image_spec(options.image) + if kind == "macos": + raise LauncherError(f"--image {value} is a macOS package, which runs only on a Mac with Apple silicon, " + f"without --runtime docker.") + gpus = query_nvidia_gpus() + amd_gpus = query_amd_gpus() + request = [] if other else args + # whether the newest images support none of the GPUs, so they run on the CPU here + gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None + wants_gpu = gpu_request(request, brun)[0] != "cpu" and bool(gpus) + image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None + # AMD GPUs (experimental): when named with --image, or when there is no NVIDIA GPU + amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) + rocm = (kind == "rocm" or (image_type or "").startswith("rocm") + or (kind in ("sif", "docker") and "rocm" in value.lower())) + if rocm and not amd_gpus: + raise LauncherError(f"The ROCm images are for AMD GPUs, and AMD's kernel driver lists none here " + f"({KFD_DEVICE} is missing), see {DOC_URL}.") + if rocm: + build = plan_amd_build(request, amd_gpus, brun, image_type or ROCM_BUILD[1]) + if kind in ("sif", "docker"): + build.prefix = "custom" + elif kind in ("auto", "version") and not gpus and amd_runs and gpu_request(request, brun)[0] != "cpu": + build = plan_amd_build(request, amd_gpus, brun) + elif kind in ("auto", "version"): + build = plan_build(request, gpus, brun) + if gpus and amd_runs: + build.notes.append("This machine also has an AMD GPU; --image rocm runs FastSurfer on it instead " + "(experimental).") + elif amd_gpus and not amd_runs: + build.notes.append(f"The ROCm image supports none of the AMD GPUs here " + f"({', '.join(map(describe_amd, amd_gpus))}).") + elif kind in ("tag", "release"): + build = plan_build_for_type(request, gpus, brun, image_type) + else: + build = Build("custom", wants_gpu) + if kind == "rocm": + kind = "auto" + + # the image: one request to Docker Hub decides about versions + hub = None if kind in ("sif", "docker") else hub_tags() + if hub is not None and kind in ("auto", "version") and build.prefix != "cpu": + newest_release = newest_version(list(hub), "cpu") + if newest_release and f"{build.prefix}-v{newest_release}" not in hub: + raise LauncherError(f"FastSurfer {newest_release} has no {build.prefix} image, which means this " + f"launcher does not know the images of the newest release. Update it with " + f"--update_launcher, or download it again from {LAUNCHER_URL}.") + local_id = local_docker_id(value) if kind == "docker" else None + # the GPUs this run uses, to tell which installed images run here; none for a run on the CPU, and + # for AMD GPUs only the ROCm image of this launcher counts + used = [g for g in gpus if build.devices is None or g.index in build.devices] if build.gpu else [] + if build.amd or build.prefix.startswith("rocm"): + used = None + plan = decide_image(kind, value, build.prefix, options.runtime, installed, + local_docker_images() if "docker" in installed else None, + local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, + used, other_program=other) + runtime = plan.runtime + if hub is not None: + stable = stable_launcher() + if stable is not None and stable != Path(__file__).resolve().read_bytes(): + info("A newer launcher is available, update it with --update_launcher.") + + if runtime != "docker": + try: + version_text = subprocess.run([runtime, "--version"], capture_output=True, text=True, + timeout=30).stdout + except (OSError, subprocess.SubprocessError): + version_text = "" + problem = runtime_too_old(version_text) + if problem is not None: + raise LauncherError(problem) + + requested = value if kind == "version" else (plan.tag if kind in ("tag", "release") else None) + if requested is not None and hub is not None: + version = requested.rsplit("-v", 1)[-1] + note = age_note(version, release_versions(list(hub), "cpu"), gpu_unsupported and build.gpu) + if note is not None: + info(note[1], level=note[0]) + for level, note in plan.notes: + info(note, level=level) + for note in build.notes: + info(note) + + # another program runs in the current directory; FastSurfer's paths are absolute already + program, workdir = (options.exec, cwd) if other else (script, None) + extra = shlex.split(options.runtime_args) + commands = [] + if runtime == "docker": + user = f"{os.getuid()}:{os.getgid()}" + # official images are built for Intel/AMD only; an own image may be built for this Mac + emulate = apple_silicon and (kind != "docker" or value.startswith(DOCKER_REPO)) + if plan.pull: + commands.append(["docker", "pull"] + (["--platform", "linux/amd64"] if emulate else []) + [plan.image]) + # Apptainer passes input and terminal on itself, Docker only when asked: another program gets the + # input when it comes from a pipe or file, and the terminal when it runs in one. Input from a + # terminal without one, e.g. a job started in the background, would stop the job. + stdin_tty = sys.stdin is not None and sys.stdin.isatty() + interactive = other and not stdin_tty + tty = other and stdin_tty and sys.stdout.isatty() + commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, + build.devices, workdir, interactive, build.amd, build.env, + amd_device_groups() if build.amd else None, tty)) + if not options.print_only: + memory = docker_memory_gib() + if memory < MIN_DOCKER_MEMORY_GIB: + where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " + "--memory 16)") if platform.system() == "Darwin" else "for Docker" + info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " + f"15 GB. Increase the memory {where}.", level="WARNING") + else: + if kind == "sif" and not Path(plan.image).is_file(): + raise LauncherError(f"--image {value}: no such file.") + if plan.build_from is not None: + commands.append([runtime, "build", plan.image, plan.build_from]) + commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, program, args, extra, + build.devices, workdir, build.amd, build.env)) + return commands, plan.record + + def main(argv: list[str] | None = None) -> int: parser = make_parser() argv, program_args = split_exec(sys.argv[1:] if argv is None else argv) @@ -1331,13 +1615,13 @@ def main(argv: list[str] | None = None) -> int: if fastsurfer_args: parser.error(f"unrecognized arguments before --exec: {' '.join(fastsurfer_args)}") fastsurfer_args = program_args or [] - elif not fastsurfer_args: + elif not fastsurfer_args and not options.update: + # --update alone downloads (and on a Mac with Apple silicon installs) the newest release parser.print_help() return 1 # the FastSurfer script to run, None for another program, which has its own flags script = RUN_SCRIPT if options.exec is None else fastsurfer_script(options.exec) other = script is None - brun = script == BRUN_SCRIPT def info(message: str, level: str = "INFO") -> None: print(f"{level}: {message}", file=sys.stderr) @@ -1345,158 +1629,19 @@ def info(message: str, level: str = "INFO") -> None: try: if os.name != "posix": raise LauncherError("Run this in WSL on Windows, where Docker's paths are Linux paths.") - # the images are built for Intel/AMD processors, which a Mac with Apple silicon only emulates + # the images are built for Intel/AMD processors, which a Mac with Apple silicon only emulates, so + # it runs the macOS package instead, unless Docker is asked for apple_silicon = is_apple_silicon() - package_hint = (f"install the FastSurfer package for macOS instead, {MACOS_PACKAGE_URL} (see {MACOS_DOC_URL}). " - f"It runs natively and uses the Mac's GPU: in the console it opens, call run_fastsurfer.sh " - f"with the same flags.") - if apple_silicon and options.runtime != "docker": - raise LauncherError(f"On a Mac with Apple silicon, {package_hint} To run the Intel image in Docker " - f"under emulation anyway, which is much slower, pass --runtime docker.") - if apple_silicon: - info(f"Docker on a Mac with Apple silicon emulates the Intel image, which is much slower than the " - f"macOS package and cannot use the GPU. Better {package_hint}", level="WARNING") - installed = {name for name in ("apptainer", "singularity", "docker") if shutil.which(name)} - if not installed: - raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") - if options.runtime != "auto" and options.runtime not in installed: - raise LauncherError(f"{options.runtime} is not installed.") - if brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): - # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get - raise LauncherError("brun_fastsurfer.sh needs the subjects in --subjects_list or " - "--subjects = ...") - - cwd = Path.cwd() - if other: - args, mounts = fastsurfer_args, exec_mounts(fastsurfer_args, cwd) - else: - fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) - args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) - mounts = minimal_mounts(mounts) - if Path("/") in mounts: - raise LauncherError("The paths would share the whole file system with the container, keep the " - "data in a directory below /.") - if Path.home() in mounts: - info("This shares your home directory with the container. Settings in it may affect " - "FastSurfer, so keep the data in a subdirectory.") - if not other and "--fs_license" not in args and "--seg_only" not in args: - info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") - - # the GPUs and the image type; another program's flags say nothing about them - kind, value = parse_image_spec(options.image) - gpus = query_nvidia_gpus() - amd_gpus = query_amd_gpus() - request = [] if other else args - # whether the newest images support none of the GPUs, so they run on the CPU here - gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None - wants_gpu = gpu_request(request, brun)[0] != "cpu" and bool(gpus) - image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None - # AMD GPUs (experimental): when named with --image, or when there is no NVIDIA GPU - amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) - rocm = (kind == "rocm" or (image_type or "").startswith("rocm") - or (kind in ("sif", "docker") and "rocm" in value.lower())) - if rocm and not amd_gpus: - raise LauncherError(f"The ROCm images are for AMD GPUs, and AMD's kernel driver lists none here " - f"({KFD_DEVICE} is missing), see {DOC_URL}.") - if rocm: - build = plan_amd_build(request, amd_gpus, brun, image_type or ROCM_BUILD[1]) - if kind in ("sif", "docker"): - build.prefix = "custom" - elif kind in ("auto", "version") and not gpus and amd_runs and gpu_request(request, brun)[0] != "cpu": - build = plan_amd_build(request, amd_gpus, brun) - elif kind in ("auto", "version"): - build = plan_build(request, gpus, brun) - if gpus and amd_runs: - build.notes.append("This machine also has an AMD GPU; --image rocm runs FastSurfer on it instead " - "(experimental).") - elif amd_gpus and not amd_runs: - build.notes.append(f"The ROCm image supports none of the AMD GPUs here " - f"({', '.join(map(describe_amd, amd_gpus))}).") - elif kind in ("tag", "release"): - build = plan_build_for_type(request, gpus, brun, image_type) - else: - build = Build("custom", wants_gpu) - if kind == "rocm": - kind = "auto" - - # the image: one request to Docker Hub decides about versions - hub = None if kind in ("sif", "docker") else hub_tags() - if hub is not None and kind in ("auto", "version") and build.prefix != "cpu": - newest_release = newest_version(list(hub), "cpu") - if newest_release and f"{build.prefix}-v{newest_release}" not in hub: - raise LauncherError(f"FastSurfer {newest_release} has no {build.prefix} image, which means this " - f"launcher does not know the images of the newest release. Update it with " - f"--update_launcher, or download it again from {LAUNCHER_URL}.") - local_id = local_docker_id(value) if kind == "docker" else None - # the GPUs this run uses, to tell which installed images run here; none for a run on the CPU, and - # for AMD GPUs only the ROCm image of this launcher counts - used = [g for g in gpus if build.devices is None or g.index in build.devices] if build.gpu else [] - if build.amd or build.prefix.startswith("rocm"): - used = None - plan = decide_image(kind, value, build.prefix, options.runtime, installed, - local_docker_images() if "docker" in installed else None, - local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, - used, other_program=other) - runtime = plan.runtime - if hub is not None: - stable = stable_launcher() - if stable is not None and stable != Path(__file__).resolve().read_bytes(): - info("A newer launcher is available, update it with --update_launcher.") - - if runtime != "docker": - try: - version_text = subprocess.run([runtime, "--version"], capture_output=True, text=True, - timeout=30).stdout - except (OSError, subprocess.SubprocessError): - version_text = "" - problem = runtime_too_old(version_text) - if problem is not None: - raise LauncherError(problem) - - requested = value if kind == "version" else (plan.tag if kind in ("tag", "release") else None) - if requested is not None and hub is not None: - version = requested.rsplit("-v", 1)[-1] - note = age_note(version, release_versions(list(hub), "cpu"), gpu_unsupported and build.gpu) - if note is not None: - info(note[1], level=note[0]) - for level, note in plan.notes: - info(note, level=level) - for note in build.notes: - info(note) - - # another program runs in the current directory; FastSurfer's paths are absolute already - program, workdir = (options.exec, cwd) if other else (script, None) - extra = shlex.split(options.runtime_args) - commands = [] - if runtime == "docker": - user = f"{os.getuid()}:{os.getgid()}" - # official images are built for Intel/AMD only; an own image may be built for this Mac - emulate = apple_silicon and (kind != "docker" or value.startswith(DOCKER_REPO)) - if plan.pull: - commands.append(["docker", "pull"] + (["--platform", "linux/amd64"] if emulate else []) + [plan.image]) - # Apptainer passes input and terminal on itself, Docker only when asked: another program gets the - # input when it comes from a pipe or file, and the terminal when it runs in one. Input from a - # terminal without one, e.g. a job started in the background, would stop the job. - stdin_tty = sys.stdin is not None and sys.stdin.isatty() - interactive = other and not stdin_tty - tty = other and stdin_tty and sys.stdout.isatty() - commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, - build.devices, workdir, interactive, build.amd, build.env, - amd_device_groups() if build.amd else None, tty)) - if not options.print_only: - memory = docker_memory_gib() - if memory < MIN_DOCKER_MEMORY_GIB: - where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " - "--memory 16)") if platform.system() == "Darwin" else "for Docker" - info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " - f"15 GB. Increase the memory {where}.", level="WARNING") + if options.runtime == "macos" and not apple_silicon: + raise LauncherError("The macOS package of FastSurfer is for Macs with Apple silicon.") + if apple_silicon and options.runtime in ("auto", "macos"): + commands, record = macos_commands(options, script, other, fastsurfer_args, info), None else: - if kind == "sif" and not Path(plan.image).is_file(): - raise LauncherError(f"--image {value}: no such file.") - if plan.build_from is not None: - commands.append([runtime, "build", plan.image, plan.build_from]) - commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, program, args, extra, - build.devices, workdir, build.amd, build.env)) + if apple_silicon: + info("Docker on a Mac with Apple silicon emulates the Intel image, which is much slower than the " + "macOS package and cannot use the GPU. Without --runtime docker, the launcher runs the package.", + level="WARNING") + commands, record = container_commands(options, script, other, fastsurfer_args, apple_silicon, info) except LauncherError as error: print(f"ERROR: {error}", file=sys.stderr) return 1 @@ -1518,11 +1663,11 @@ def info(message: str, level: str = "INFO") -> None: "`apptainer registry list` and $APPTAINER_DOCKER_USERNAME.", file=sys.stderr) return 1 - record = Path(command[2]).with_name(Path(command[2]).name + ".digest") - if plan.record: - record.write_text(plan.record + "\n") - elif record.exists(): - record.unlink() + digest_file = Path(command[2]).with_name(Path(command[2]).name + ".digest") + if record: + digest_file.write_text(record + "\n") + elif digest_file.exists(): + digest_file.unlink() elif subprocess.run(command).returncode != 0: print(f"ERROR: {shlex.join(command)} failed.", file=sys.stderr) return 1 From eb8f1bbf2af2482dcae17f23f36c3108ffd48377 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 09:20:41 +0200 Subject: [PATCH 11/15] Simplify the quick start to one set of commands and say when to run FastSurfer directly --- doc/overview/QUICKSTART.md | 47 +++++++++++++++----------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index 2b79340e3..f9895aeef 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -1,8 +1,15 @@ Quick Start =========== -This example segments one brain MRI in a few minutes, with the same commands on Linux, Windows and macOS. You can -also run it without installing anything in [Google Colab](#google-colab). The [installation guide](INSTALL.md) -explains the options in more detail. +This example segments one brain MRI in a few minutes with `fastsurfer`, a launcher that runs FastSurfer with the +same commands on Linux, Windows and macOS. It takes the flags of `run_fastsurfer.sh` and sets up the rest: on Linux, +Windows and Intel Macs, it picks, downloads and runs the right container image; on a Mac with Apple silicon, it runs +the installed macOS package. You can also try FastSurfer without installing anything in +[Google Colab](#google-colab). + +The launcher is meant for getting started and for single images. For larger projects, such as whole datasets or +runs on an HPC cluster, we recommend running FastSurfer directly, which gives full control over images, paths and +resources: see the [installation guide](INSTALL.md), [Running FastSurfer in a container](CONTAINERS.md), and the +scripts for [many subjects](../scripts/BATCH.md) and for [SLURM clusters](../scripts/SLURM.md). Segment an example image ------------------------ @@ -10,30 +17,15 @@ With a GPU (8 GB of graphics memory) this takes well under a minute; without one a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 mm voxel sizes, no preprocessing, `.nii`, `.nii.gz` or `.mgz`), or the example image the commands download. -First, install what runs FastSurfer on your system: - -`````{tab-set} - -````{tab-item} Linux, Windows or Intel Mac -:sync: container - -A working installation of Apptainer, Singularity or Docker, which the commands below do not set up: see -[Linux](install/LINUX.md), [Windows](install/WINDOWS.md) (Docker in WSL2, run the commands there) or -[Intel Macs](install/MACOS.md#docker-intel-macs). `docker run --rm hello-world` or `apptainer --version` checks it. -The launcher then picks the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU), downloads it on the -first run, and shares this directory with the container; [Running FastSurfer in a container](CONTAINERS.md) shows -the commands it runs. -```` +First, install what the launcher runs FastSurfer with: -````{tab-item} Mac with Apple silicon -:sync: macos - -The [macOS package](install/MACOS.md) (Macs with M1 or newer), which runs natively and uses the Mac's GPU. Once -you have downloaded the launcher (step 1 below), `./fastsurfer --update` downloads the package and opens the macOS -Installer. The launcher then runs the installed package. -```` - -````` +- **Linux, Windows or Intel Mac:** Apptainer, Singularity or Docker, see [Linux](install/LINUX.md), + [Windows](install/WINDOWS.md) (Docker in WSL2, run the commands below there) or + [Intel Macs](install/MACOS.md#docker-intel-macs). `docker run --rm hello-world` or `apptainer --version` checks + it. The launcher picks the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU) and downloads it on + the first run. +- **Mac with Apple silicon (M1 or newer):** the [macOS package](install/MACOS.md), which runs natively and uses the + Mac's GPU. After step 1 below, `./fastsurfer --update` downloads it and opens the macOS Installer. Then download the launcher and run FastSurfer: @@ -57,9 +49,6 @@ curl -k \ --seg_only --no_biasfield --no_cereb --no_hypothal ``` -The [launcher](CONTAINERS.md#the-launcher) takes the flags of `run_fastsurfer.sh`, which you can also call -directly in the Terminal window of the FastSurfer app on a Mac. - That's it, it will run the full brain segmentation. For speed, we switched off the cerebellum and hypothalamic sub-segmentation (would add a couple minutes). We also switched off the bias field correction, which is used to compute partial volume estimates for the statsfiles, so you might want to switch it on again if you want the volume statistics text file (under ```subjectX/stats```). Also if you need the estimated total intracranial volume for correcting the stats, you would either need to run the surface stream or switch on the Talairach registration with From 74764b29cde5fb9b48944d146b29e4a00c110de9 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 09:25:54 +0200 Subject: [PATCH 12/15] Link the launcher download to the copy of the same docs tree --- doc/CONVENTIONS.md | 4 +++- doc/conf.py | 2 ++ doc/overview/CONTAINERS.md | 2 +- doc/overview/QUICKSTART.md | 2 +- doc/overview/install/LINUX.md | 2 +- doc/overview/install/MACOS.md | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/CONVENTIONS.md b/doc/CONVENTIONS.md index a330b8ed6..93b95a4cf 100644 --- a/doc/CONVENTIONS.md +++ b/doc/CONVENTIONS.md @@ -157,7 +157,9 @@ supported versions. `doc/conf.py`), with a separate text for `{{ FASTSURFER_VERSION }}`, `{{ CUDA_STRING }}` and both: that the commands use the latest release (in the development documentation), and that images for other CUDA versions are available on Docker Hub. Do not add such notes by hand. Where the version is not about an image, such as the folder of the macOS -package, use `{{ PACKAGE_VERSION }}`, the same version without the note. +package, use `{{ PACKAGE_VERSION }}`, the same version without the note. The download of the launcher uses +`{{ LAUNCHER_URL }}`, the copy published next to the pages of the same tree (`stable` or `dev`). `README.md` is +shown on GitHub, where nothing is substituted, so it names the `stable` copy. What this tree builds comes from this tree, not from that release: `{{ CUDA_DEFAULT_STRING }}` and `{{ ROCM_DEFAULT_STRING }}` (with `{{ CUDA_DEFAULT_VERSION }}` and `{{ ROCM_DEFAULT_VERSION }}`) are the default CUDA diff --git a/doc/conf.py b/doc/conf.py index d6e821add..9858fb99b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -209,6 +209,8 @@ def _tool_values_gitref(ref: str | None, *keys: str) -> tuple[str, ...]: "PYTHON_VERSION": version_python, "UBUNTU_VERSION": version_ubuntu, "FREESURFER_VERSION": version_freesurfer, + # the launcher published next to this tree's pages (html_extra_path), so each tree links to its own copy + "LAUNCHER_URL": f"https://deep-mi.org/FastSurfer/{'stable' if documents_a_release else 'dev'}/fastsurfer", } templates_path = ["_templates"] diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index c76776dbe..11ab1f53a 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -39,7 +39,7 @@ The launcher `fastsurfer` writes and runs the container command for you. It take Download it, and run FastSurfer: ```bash -curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer +curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer ./fastsurfer \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ --sid subjectX --sd $HOME/my_fastsurfer_analysis \ diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index f9895aeef..3abcf5f02 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -35,7 +35,7 @@ mkdir fastsurfer_test cd fastsurfer_test # 1. Download the launcher (only the first time) -curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer +curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer # 2. Download an example brain MRI (if you don't have your own) # If you have your own, copy it to this directory and adjust diff --git a/doc/overview/install/LINUX.md b/doc/overview/install/LINUX.md index 6f09506a7..3ee6c1eb9 100644 --- a/doc/overview/install/LINUX.md +++ b/doc/overview/install/LINUX.md @@ -14,7 +14,7 @@ picks the image for your NVIDIA GPU and driver (or for an AMD GPU, experimental, and runs FastSurfer with the flags of `run_fastsurfer.sh`: ```bash -curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer +curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer ./fastsurfer --t1 --sid --sd ``` diff --git a/doc/overview/install/MACOS.md b/doc/overview/install/MACOS.md index a66be79ab..7b94f3dfc 100644 --- a/doc/overview/install/MACOS.md +++ b/doc/overview/install/MACOS.md @@ -76,7 +76,7 @@ Without opening the console, the [launcher](../CONTAINERS.md#the-launcher) `fast from any Terminal window, with the same flags and the same commands as on Linux: ```bash -curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer +curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer ./fastsurfer --seg_only --sd $HOME/my_fastsurfer_analysis --sid subjectX \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz ``` From c892cfa73c7544a127468c6427dd337f0dd03983 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 09:36:53 +0200 Subject: [PATCH 13/15] Keep converted custom images apart from release images, give own images the AMD GPUs, and pass --help to brun_fastsurfer.sh --- test/utils/test_fastsurfer_launcher.py | 25 +++++++- tools/fastsurfer | 79 ++++++++++++++++---------- 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/test/utils/test_fastsurfer_launcher.py b/test/utils/test_fastsurfer_launcher.py index 2359ad258..ed92d0c40 100644 --- a/test/utils/test_fastsurfer_launcher.py +++ b/test/utils/test_fastsurfer_launcher.py @@ -337,12 +337,17 @@ def test_decide_image_named(tmp_path): assert decide(tmp_path, kind="docker", value="my_fs:cu132", local_id="sha256:1").runtime == "docker" plan = decide(tmp_path, kind="docker", value="my_fs:cu132", runtime="apptainer", local_id="sha256:1") assert (plan.build_from, plan.record) == ("docker-daemon://my_fs:cu132", "sha256:1") - (tmp_path / "my_fs-cu132.sif").write_bytes(b"") - (tmp_path / "my_fs-cu132.sif.digest").write_text("sha256:1\n") + assert plan.image == str(tmp_path / "docker-my_fs-cu132.sif") + (tmp_path / "docker-my_fs-cu132.sif").write_bytes(b"") + (tmp_path / "docker-my_fs-cu132.sif.digest").write_text("sha256:1\n") own = {"kind": "docker", "value": "my_fs:cu132", "runtime": "apptainer"} assert decide(tmp_path, **own, local_id="sha256:1").build_from is None # a changed local Docker image is converted again assert decide(tmp_path, **own, local_id="sha256:2").build_from + # one whose name ends like a release is not taken for the installed release image, nor replaces it + (tmp_path / "fastsurfer-cu132-v2.6.0.sif").write_bytes(b"") + plan = decide(tmp_path, kind="docker", value="myrepo/fastsurfer:cu132-v2.6.0", runtime="apptainer") + assert plan.image == str(tmp_path / "docker-myrepo_fastsurfer-cu132-v2.6.0.sif") and plan.build_from def test_plan_paths_run(tmp_path): @@ -559,6 +564,22 @@ def test_plan_amd_build(): launcher.plan_amd_build(["--device", "cuda"], [old], False) +def test_plan_image_build_custom(): + nvidia, amd = [gpu((8, 6))], [launcher.AmdGpu(0, (11, 0, 0), 24)] + # an own image on a machine with only AMD GPUs gets them, whatever its name + build = launcher.plan_image_build("docker", "my_fastsurfer:dev", [], [], amd, False) + assert (build.prefix, build.gpu, build.amd) == ("custom", True, True) + assert launcher.plan_image_build("sif", "/images/own.sif", [], [], amd, False).amd + # with an NVIDIA GPU, the NVIDIA one, unless the name says ROCm + build = launcher.plan_image_build("docker", "my_fastsurfer:dev", [], nvidia, amd, False) + assert (build.gpu, build.amd) == (True, False) + assert launcher.plan_image_build("docker", "my_fastsurfer:rocm-dev", [], nvidia, amd, False).amd + # a ROCm image without an AMD GPU stops, and --device cpu gives the container no GPU + with pytest.raises(launcher.LauncherError, match="AMD"): + launcher.plan_image_build("docker", "my_fastsurfer:rocm-dev", [], nvidia, [], False) + assert not launcher.plan_image_build("docker", "my_fastsurfer:dev", ["--device", "cpu"], [], amd, False).gpu + + def test_decide_image_rocm(tmp_path): # for an AMD GPU only the ROCm image counts among the installed ones, else the newest ROCm release hub = {**HUB, "rocm7.2-v2.6.0": {"sha256:r"}} diff --git a/tools/fastsurfer b/tools/fastsurfer index 7911dd08c..b2eb21623 100755 --- a/tools/fastsurfer +++ b/tools/fastsurfer @@ -560,6 +560,44 @@ def plan_amd_build(args: list[str], gpus: list[AmdGpu], brun: bool, image_type: return Build(image_type, True, notes, devices, amd=True, env=env) +def plan_image_build(kind: str, value: str, request: list[str], gpus: list[Gpu], amd_gpus: list[AmdGpu], + brun: bool) -> Build: + """ + The image type and the GPUs for what --image names (see parse_image_spec) and these arguments. + + AMD GPUs (experimental) are used for a ROCm image, named with --image, and when there is no NVIDIA GPU. + An own image (a Docker image or .sif file) gets the GPUs of this machine: the AMD ones if its name says + ROCm or there is no NVIDIA GPU, else the NVIDIA ones. + """ + custom = kind in ("sif", "docker") + image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None + wants_gpu = gpu_request(request, brun)[0] != "cpu" + amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) + rocm = kind == "rocm" or (image_type or "").startswith("rocm") or (custom and "rocm" in value.lower()) + if rocm and not amd_gpus: + raise LauncherError(f"The ROCm images are for AMD GPUs, and AMD's kernel driver lists none here " + f"({KFD_DEVICE} is missing), see {DOC_URL}.") + if rocm or (custom and amd_gpus and not gpus and wants_gpu): + build = plan_amd_build(request, amd_gpus, brun, image_type or ROCM_BUILD[1]) + if custom: + build.prefix = "custom" + return build + if custom: + return Build("custom", wants_gpu and bool(gpus)) + if kind in ("tag", "release"): + return plan_build_for_type(request, gpus, brun, image_type) + if not gpus and amd_runs and wants_gpu: + return plan_amd_build(request, amd_gpus, brun) + build = plan_build(request, gpus, brun) + if gpus and amd_runs: + build.notes.append("This machine also has an AMD GPU; --image rocm runs FastSurfer on it instead " + "(experimental).") + elif amd_gpus and not amd_runs: + build.notes.append(f"The ROCm image supports none of the AMD GPUs here " + f"({', '.join(map(describe_amd, amd_gpus))}).") + return build + + def amd_device_groups(device: Path = KFD_DEVICE, dri: Path = Path("/dev/dri")) -> list[int]: """The groups that own the devices of AMD GPUs, which a Docker container running as the user needs.""" # the render nodes of the GPUs @@ -663,6 +701,14 @@ def sif_path(image_dir: Path, tag: str) -> Path: return image_dir / f"{SIF_PREFIX}{tag}.sif" +def converted_sif(image_dir: Path, reference: str) -> Path: + """ + Where the Apptainer image converted from another Docker image is kept, named after its full reference, + e.g. docker-myrepo_fastsurfer-cu132.sif, so it cannot take the place of an official release image. + """ + return image_dir / f"docker-{re.sub(r'[/:@]', lambda m: '_' if m.group() == '/' else '-', reference)}.sif" + + def _recorded(sif: Path) -> str: """What the launcher wrote next to an image it built: the digest or image ID of its source, or "".""" record = sif.with_name(sif.name + ".digest") @@ -830,7 +876,7 @@ def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: se if runtime == "docker" or (runtime == "auto" and (local_id is not None or apptainer is None)): return ImagePlan(require("docker"), value, pull=update) # Apptainer runs a Docker image as a .sif file, converted from the local Docker if it is there - sif = image_dir / (value.rsplit("/", 1)[-1].replace(":", "-") + ".sif") + sif = converted_sif(image_dir, value) source = f"docker-daemon://{value}" if local_id else f"docker://{value}" stale = sif.is_file() and local_id is not None and _recorded(sif) != local_id build = update or not sif.is_file() or stale @@ -1463,7 +1509,7 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") if options.runtime != "auto" and options.runtime not in installed: raise LauncherError(f"{options.runtime} is not installed.") - if brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): + if brun and not {"--subjects", "--subjects_list", "--subject_list", "--help"} & set(fastsurfer_args): # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get raise LauncherError("brun_fastsurfer.sh needs the subjects in --subjects_list or " "--subjects = ...") @@ -1490,37 +1536,10 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b raise LauncherError(f"--image {value} is a macOS package, which runs only on a Mac with Apple silicon, " f"without --runtime docker.") gpus = query_nvidia_gpus() - amd_gpus = query_amd_gpus() request = [] if other else args # whether the newest images support none of the GPUs, so they run on the CPU here gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None - wants_gpu = gpu_request(request, brun)[0] != "cpu" and bool(gpus) - image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None - # AMD GPUs (experimental): when named with --image, or when there is no NVIDIA GPU - amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) - rocm = (kind == "rocm" or (image_type or "").startswith("rocm") - or (kind in ("sif", "docker") and "rocm" in value.lower())) - if rocm and not amd_gpus: - raise LauncherError(f"The ROCm images are for AMD GPUs, and AMD's kernel driver lists none here " - f"({KFD_DEVICE} is missing), see {DOC_URL}.") - if rocm: - build = plan_amd_build(request, amd_gpus, brun, image_type or ROCM_BUILD[1]) - if kind in ("sif", "docker"): - build.prefix = "custom" - elif kind in ("auto", "version") and not gpus and amd_runs and gpu_request(request, brun)[0] != "cpu": - build = plan_amd_build(request, amd_gpus, brun) - elif kind in ("auto", "version"): - build = plan_build(request, gpus, brun) - if gpus and amd_runs: - build.notes.append("This machine also has an AMD GPU; --image rocm runs FastSurfer on it instead " - "(experimental).") - elif amd_gpus and not amd_runs: - build.notes.append(f"The ROCm image supports none of the AMD GPUs here " - f"({', '.join(map(describe_amd, amd_gpus))}).") - elif kind in ("tag", "release"): - build = plan_build_for_type(request, gpus, brun, image_type) - else: - build = Build("custom", wants_gpu) + build = plan_image_build(kind, value, request, gpus, query_amd_gpus(), brun) if kind == "rocm": kind = "auto" From 4a386a5232799d378c0bf8697104526999036d11 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sat, 17 Oct 2026 10:13:34 +0200 Subject: [PATCH 14/15] Let --update alone only download, run the ROCm image on the CPU, and document the launcher on WSL --- README.md | 3 +- doc/overview/CONTAINERS.md | 17 ++++++++---- doc/overview/QUICKSTART.md | 3 +- doc/overview/install/LINUX.md | 6 ++-- doc/overview/install/MACOS.md | 3 +- doc/overview/install/WINDOWS.md | 21 +++++++++++++- test/utils/test_fastsurfer_launcher.py | 16 +++++++++-- tools/fastsurfer | 38 ++++++++++++++++++-------- 8 files changed, 82 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 2c81e9ada..d5aef93ca 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,8 @@ All installation methods use the `run_fastsurfer.sh` call interface (replace the 1. For container installations, the [launcher](doc/overview/CONTAINERS.md#the-launcher) is the easiest way: it takes the `` and writes and runs the container command, with the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU): ```bash - curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer && chmod +x fastsurfer + curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer + chmod +x fastsurfer ./fastsurfer ``` diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 11ab1f53a..3c1d7f5b8 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -39,7 +39,8 @@ The launcher `fastsurfer` writes and runs the container command for you. It take Download it, and run FastSurfer: ```bash -curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer ./fastsurfer \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ --sid subjectX --sd $HOME/my_fastsurfer_analysis \ @@ -67,17 +68,23 @@ and are handled like `run_fastsurfer.sh`: ```bash # several subjects; the paths in a subjects list have to be absolute -./fastsurfer --exec brun_fastsurfer.sh --subjects_list subjects.txt --sd results +./fastsurfer --exec brun_fastsurfer.sh \ + --subjects_list $HOME/my_mri_data/subjects_list.txt \ + --sd $HOME/my_fastsurfer_analysis # the time points of one person (longitudinal processing) -./fastsurfer --exec long_fastsurfer.sh --tid person --t1s tp1.nii.gz tp2.nii.gz \ - --tpids tp1 tp2 --sd results +./fastsurfer --exec long_fastsurfer.sh --tid subjectX \ + --t1s $HOME/my_mri_data/subjectX/t1_weighted_20210315.nii.gz \ + $HOME/my_mri_data/subjectX/t1_weighted_20230920.nii.gz \ + --tpids subjectX_20210315 subjectX_20230920 \ + --sd $HOME/my_fastsurfer_analysis ``` ([long_fastsurfer.sh](../scripts/long_fastsurfer.rst) explains longitudinal processing.) Any other program runs in the current directory, for example to convert a segmentation to NIfTI: ```bash -./fastsurfer --exec nib-convert subjectX/mri/aparc.DKTatlas+aseg.deep.mgz aparc.nii.gz +./fastsurfer --exec nib-convert \ + subjectX/mri/aparc.DKTatlas+aseg.deep.mgz aparc.nii.gz ``` The launcher shares the current directory with the container, as it does the directories of absolute diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index 3abcf5f02..32b1cb399 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -35,7 +35,8 @@ mkdir fastsurfer_test cd fastsurfer_test # 1. Download the launcher (only the first time) -curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer # 2. Download an example brain MRI (if you don't have your own) # If you have your own, copy it to this directory and adjust diff --git a/doc/overview/install/LINUX.md b/doc/overview/install/LINUX.md index 3ee6c1eb9..0708b488d 100644 --- a/doc/overview/install/LINUX.md +++ b/doc/overview/install/LINUX.md @@ -14,8 +14,10 @@ picks the image for your NVIDIA GPU and driver (or for an AMD GPU, experimental, and runs FastSurfer with the flags of `run_fastsurfer.sh`: ```bash -curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer -./fastsurfer --t1 --sid --sd +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer +./fastsurfer --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ + --sid subjectX --sd $HOME/my_fastsurfer_analysis ``` The sections below explain the images and how to download them by hand. diff --git a/doc/overview/install/MACOS.md b/doc/overview/install/MACOS.md index 7b94f3dfc..9a6b2e5e4 100644 --- a/doc/overview/install/MACOS.md +++ b/doc/overview/install/MACOS.md @@ -76,7 +76,8 @@ Without opening the console, the [launcher](../CONTAINERS.md#the-launcher) `fast from any Terminal window, with the same flags and the same commands as on Linux: ```bash -curl -fsSLO {{ LAUNCHER_URL }} && chmod +x fastsurfer +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer ./fastsurfer --seg_only --sd $HOME/my_fastsurfer_analysis --sid subjectX \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz ``` diff --git a/doc/overview/install/WINDOWS.md b/doc/overview/install/WINDOWS.md index 5061d2c46..9d565d413 100644 --- a/doc/overview/install/WINDOWS.md +++ b/doc/overview/install/WINDOWS.md @@ -9,7 +9,25 @@ Make sure that these are installed and running: * [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) * [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/) -Then choose the image for your hardware. Run the commands in Windows PowerShell. +For a GPU, the requirements in the NVIDIA GPU tab below apply as well. + +The [launcher](../CONTAINERS.md#the-launcher) writes the Docker command for you. Run it in the terminal of your +WSL2 Linux distribution (for example Ubuntu, from the Start menu), not in PowerShell. Docker Desktop's `docker` +command is available there once you switch on the WSL integration for that distribution (Docker Desktop: +**Settings > Resources > WSL integration**). For example: + +```bash +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer +./fastsurfer --seg_only --sd $HOME/my_fastsurfer_analysis --sid subjectX \ + --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz +``` + +It picks the image for your NVIDIA GPU (or the CPU image), downloads it, shares the folders with the container and +runs it as your WSL user. Keep the data in your WSL home folder: Windows folders are available below `/mnt/c/`, but +reading them from WSL is slower. + +Without the launcher, choose the image for your hardware and run the commands in Windows PowerShell. `````{tab-set} @@ -79,3 +97,4 @@ Linux user IDs anyway, so pass a fixed user ID instead, for example `1000:1000`. Docker Desktop reserves only part of your memory for WSL2. FastSurfer needs at least the memory in the [system requirements](../intro.rst#system-requirements); if it fails, check [how much memory your WSL2 distribution has](https://www.aleksandrhovhannisyan.com/blog/limiting-memory-usage-in-wsl-2/). +The launcher warns if Docker has less, and the limit is set in `%UserProfile%\.wslconfig`. diff --git a/test/utils/test_fastsurfer_launcher.py b/test/utils/test_fastsurfer_launcher.py index ed92d0c40..24af4ff94 100644 --- a/test/utils/test_fastsurfer_launcher.py +++ b/test/utils/test_fastsurfer_launcher.py @@ -574,12 +574,22 @@ def test_plan_image_build_custom(): build = launcher.plan_image_build("docker", "my_fastsurfer:dev", [], nvidia, amd, False) assert (build.gpu, build.amd) == (True, False) assert launcher.plan_image_build("docker", "my_fastsurfer:rocm-dev", [], nvidia, amd, False).amd - # a ROCm image without an AMD GPU stops, and --device cpu gives the container no GPU - with pytest.raises(launcher.LauncherError, match="AMD"): - launcher.plan_image_build("docker", "my_fastsurfer:rocm-dev", [], nvidia, [], False) + # --device cpu gives the container no GPU assert not launcher.plan_image_build("docker", "my_fastsurfer:dev", ["--device", "cpu"], [], amd, False).gpu +def test_plan_image_build_rocm_on_cpu(): + # an installed ROCm image also runs on the CPU, without an AMD GPU, with a note unless the CPU was asked for + for kind, value in (("rocm", ""), ("tag", "rocm7.2-v2.6.0")): + build = launcher.plan_image_build(kind, value, [], [gpu((8, 6))], [], False) + assert (build.prefix, build.gpu, build.amd) == ("rocm7.2", False, False) and "CPU" in build.notes[0] + build = launcher.plan_image_build(kind, value, ["--device", "cpu"], [], [], False) + assert (build.prefix, build.gpu, build.notes) == ("rocm7.2", False, []) + # a GPU requested for it without an AMD GPU stops + with pytest.raises(launcher.LauncherError, match="No AMD GPU"): + launcher.plan_image_build("rocm", "", ["--device", "cuda"], [], [], False) + + def test_decide_image_rocm(tmp_path): # for an AMD GPU only the ROCm image counts among the installed ones, else the newest ROCm release hub = {**HUB, "rocm7.2-v2.6.0": {"sha256:r"}} diff --git a/tools/fastsurfer b/tools/fastsurfer index b2eb21623..bfff4ed6f 100755 --- a/tools/fastsurfer +++ b/tools/fastsurfer @@ -566,17 +566,15 @@ def plan_image_build(kind: str, value: str, request: list[str], gpus: list[Gpu], The image type and the GPUs for what --image names (see parse_image_spec) and these arguments. AMD GPUs (experimental) are used for a ROCm image, named with --image, and when there is no NVIDIA GPU. - An own image (a Docker image or .sif file) gets the GPUs of this machine: the AMD ones if its name says - ROCm or there is no NVIDIA GPU, else the NVIDIA ones. + A ROCm image without a usable AMD GPU runs on the CPU, unless a GPU is requested. An own image (a Docker + image or .sif file) gets the GPUs of this machine: the AMD ones if its name says ROCm or there is no + NVIDIA GPU, else the NVIDIA ones. """ custom = kind in ("sif", "docker") image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None wants_gpu = gpu_request(request, brun)[0] != "cpu" amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) rocm = kind == "rocm" or (image_type or "").startswith("rocm") or (custom and "rocm" in value.lower()) - if rocm and not amd_gpus: - raise LauncherError(f"The ROCm images are for AMD GPUs, and AMD's kernel driver lists none here " - f"({KFD_DEVICE} is missing), see {DOC_URL}.") if rocm or (custom and amd_gpus and not gpus and wants_gpu): build = plan_amd_build(request, amd_gpus, brun, image_type or ROCM_BUILD[1]) if custom: @@ -1504,6 +1502,8 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b to record next to an Apptainer image they build (None if they build none). """ brun = script == BRUN_SCRIPT + # --update without FastSurfer's flags only downloads (or builds) the image + download_only = options.update and not other and not fastsurfer_args installed = {name for name in ("apptainer", "singularity", "docker") if shutil.which(name)} if not installed: raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") @@ -1527,7 +1527,7 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b if Path.home() in mounts: info("This shares your home directory with the container. Settings in it may affect " "FastSurfer, so keep the data in a subdirectory.") - if not other and "--fs_license" not in args and "--seg_only" not in args: + if not other and not download_only and "--fs_license" not in args and "--seg_only" not in args: info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") # the GPUs and the image type; another program's flags say nothing about them @@ -1560,7 +1560,7 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b plan = decide_image(kind, value, build.prefix, options.runtime, installed, local_docker_images() if "docker" in installed else None, local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, - used, other_program=other) + used, other_program=other or download_only) runtime = plan.runtime if hub is not None: stable = stable_launcher() @@ -1607,11 +1607,18 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, build.devices, workdir, interactive, build.amd, build.env, amd_device_groups() if build.amd else None, tty)) - if not options.print_only: + if not options.print_only and not download_only: memory = docker_memory_gib() if memory < MIN_DOCKER_MEMORY_GIB: - where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " - "--memory 16)") if platform.system() == "Darwin" else "for Docker" + if platform.system() == "Darwin": + where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " + "--memory 16)") + elif "microsoft" in platform.release().lower(): + # Docker Desktop on WSL2 runs in the memory WSL2 has + where = ("of WSL2 in %UserProfile%\\.wslconfig (memory=16GB in the [wsl2] section), then run " + "wsl --shutdown") + else: + where = "for Docker" info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " f"15 GB. Increase the memory {where}.", level="WARNING") else: @@ -1621,6 +1628,10 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b commands.append([runtime, "build", plan.image, plan.build_from]) commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, program, args, extra, build.devices, workdir, build.amd, build.env)) + if download_only: + commands = commands[:-1] + if not commands: + raise LauncherError(f"--update has nothing to download for {plan.image}.") return commands, plan.record @@ -1669,7 +1680,9 @@ def main(argv: list[str] | None = None) -> int: for command in commands: print(shlex.join(command)) return 0 - for command in commands[:-1]: + # --update alone runs only the download steps, otherwise the last command is FastSurfer (or --exec) + download_only = options.update and options.exec is None and not fastsurfer_args + for command in commands if download_only else commands[:-1]: if command[1] == "build": info(f"Building the image once: {shlex.join(command)}") # the official images are public, so their download needs no login @@ -1690,6 +1703,9 @@ def main(argv: list[str] | None = None) -> int: elif subprocess.run(command).returncode != 0: print(f"ERROR: {shlex.join(command)} failed.", file=sys.stderr) return 1 + if download_only: + info("Done. Run FastSurfer with the same command and its flags, without --update.") + return 0 info(f"Running: {shlex.join(commands[-1])}") sys.stdout.flush() sys.stderr.flush() From b09dfcc6bb884b2e8abd8464c1266e5a5060d286 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sun, 18 Oct 2026 12:12:05 +0200 Subject: [PATCH 15/15] Set up each system in the quick start and let the launcher say what is missing --- doc/CONVENTIONS.md | 7 +++ doc/conf.py | 2 + doc/overview/INSTALL.md | 5 +- doc/overview/QUICKSTART.md | 65 ++++++++++++++++---- doc/overview/install/LINUX.md | 12 +++- doc/overview/install/MACOS.md | 19 +++--- doc/overview/install/WINDOWS.md | 18 +++--- doc/overview/install/setup/linux.md | 19 ++++++ doc/overview/install/setup/macos_intel.md | 7 +++ doc/overview/install/setup/windows.md | 16 +++++ test/utils/test_fastsurfer_launcher.py | 20 ++++++ tools/fastsurfer | 75 +++++++++++++++++++---- 12 files changed, 220 insertions(+), 45 deletions(-) create mode 100644 doc/overview/install/setup/linux.md create mode 100644 doc/overview/install/setup/macos_intel.md create mode 100644 doc/overview/install/setup/windows.md diff --git a/doc/CONVENTIONS.md b/doc/CONVENTIONS.md index 93b95a4cf..88da68214 100644 --- a/doc/CONVENTIONS.md +++ b/doc/CONVENTIONS.md @@ -129,6 +129,13 @@ Names are snake_case, except `subjectX` and the file names of container images. example `-v C:/Users/user/my_mri_data:/home/user/my_mri_data`. The license file is mounted from its made-up path: `-v C:/path/to/your/freesurfer/license_file:/home/user/freesurfer_license`. +## Shared setup + +What each system needs before FastSurfer runs (a container runtime, WSL2, Docker Desktop) is written once, in +`doc/overview/install/setup/`, and included in the quick start and the install page of that system with +`:relative-docs: .`. Links in these files are relative to that folder. `exclude_patterns` in `doc/conf.py` keeps them +from becoming pages of their own. + ## Line length Lines in code fences in the documentation are at most 80 characters, so there is no scroll bar. Count the rendered diff --git a/doc/conf.py b/doc/conf.py index 9858fb99b..3996eceb1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -222,6 +222,8 @@ def _tool_values_gitref(ref: str | None, *keys: str) -> tuple[str, ...]: # instructions for writing the documentation, not part of it "AGENTS.md", "CONVENTIONS.md", + # the setup of each system, included in the quick start and the install pages rather than pages of their own + "overview/install/setup/*", ] diff --git a/doc/overview/INSTALL.md b/doc/overview/INSTALL.md index d51b2fbff..661bd1a86 100644 --- a/doc/overview/INSTALL.md +++ b/doc/overview/INSTALL.md @@ -1,7 +1,8 @@ Installation ============ -FastSurfer works the same way on every system: you call `run_fastsurfer.sh` with a T1-weighted MRI image. How you -install it depends on your system. Pick yours: +FastSurfer works the same way on every system: you call `run_fastsurfer.sh` with a T1-weighted MRI image. To try it, +the [quick start](QUICKSTART.md#set-up-your-computer) sets up your computer and runs FastSurfer with the launcher in +a few steps. How you install it for your own work depends on your system. Pick yours: ````{grid} 1 2 2 2 :gutter: 3 diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index 32b1cb399..1705705fc 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -11,23 +11,62 @@ runs on an HPC cluster, we recommend running FastSurfer directly, which gives fu resources: see the [installation guide](INSTALL.md), [Running FastSurfer in a container](CONTAINERS.md), and the scripts for [many subjects](../scripts/BATCH.md) and for [SLURM clusters](../scripts/SLURM.md). +Set up your computer +-------------------- +Once, set up what the launcher runs FastSurfer with: + +`````{tab-set} + +````{tab-item} Linux +:sync: linux + +```{include} install/setup/linux.md +:relative-docs: . +``` + +The launcher picks the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU) and downloads it on the +first run. [Linux](install/LINUX.md) explains the images. +```` + +````{tab-item} Windows +:sync: windows + +```{include} install/setup/windows.md +:relative-docs: . +``` + +[Windows](install/WINDOWS.md) has more details. +```` + +````{tab-item} Mac with Apple silicon +:sync: macos + +FastSurfer runs natively with the [macOS package](install/MACOS.md), which uses the Mac's GPU (M1 or newer). +Download and install it with the button on that page, or let the launcher do it: after step 1 below, +`./fastsurfer --update` downloads the package and opens the macOS Installer. macOS may block the installer, because +it is not signed by Apple yet; [Install](install/MACOS.md#install) explains how to allow it. +```` + +````{tab-item} Intel Mac +:sync: macos-intel + +FastSurfer runs in Docker, on the CPU (the macOS package needs Apple silicon): + +```{include} install/setup/macos_intel.md +:relative-docs: . +``` +```` + +````` + +If the launcher still cannot run FastSurfer, it says what is missing and how to fix it. + Segment an example image ------------------------ With a GPU (8 GB of graphics memory) this takes well under a minute; without one, FastSurfer uses the CPU and takes a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 mm voxel sizes, no preprocessing, -`.nii`, `.nii.gz` or `.mgz`), or the example image the commands download. - -First, install what the launcher runs FastSurfer with: - -- **Linux, Windows or Intel Mac:** Apptainer, Singularity or Docker, see [Linux](install/LINUX.md), - [Windows](install/WINDOWS.md) (Docker in WSL2, run the commands below there) or - [Intel Macs](install/MACOS.md#docker-intel-macs). `docker run --rm hello-world` or `apptainer --version` checks - it. The launcher picks the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU) and downloads it on - the first run. -- **Mac with Apple silicon (M1 or newer):** the [macOS package](install/MACOS.md), which runs natively and uses the - Mac's GPU. After step 1 below, `./fastsurfer --update` downloads it and opens the macOS Installer. - -Then download the launcher and run FastSurfer: +`.nii`, `.nii.gz` or `.mgz`), or the example image the commands download. On Windows, run the commands in the +Ubuntu terminal. ```bash # 0. Create a directory for this test diff --git a/doc/overview/install/LINUX.md b/doc/overview/install/LINUX.md index 0708b488d..9641d3aae 100644 --- a/doc/overview/install/LINUX.md +++ b/doc/overview/install/LINUX.md @@ -9,9 +9,15 @@ and for the CPU only, on [Docker Hub](https://hub.docker.com/r/deepmi/fastsurfer A GPU with enough memory makes the segmentation much faster, see the [system requirements](../intro.rst#system-requirements). Without a GPU, use the [CPU image](#cpu-only). -With Apptainer, Singularity or Docker installed, the [launcher](../CONTAINERS.md#the-launcher) does the rest: it -picks the image for your NVIDIA GPU and driver (or for an AMD GPU, experimental, or the CPU image), downloads it, -and runs FastSurfer with the flags of `run_fastsurfer.sh`: +Set up +------ +```{include} setup/linux.md +:relative-docs: . +``` + +Then the [launcher](../CONTAINERS.md#the-launcher) does the rest: it picks the image for your NVIDIA GPU and driver +(or for an AMD GPU, experimental, or the CPU image), downloads it, and runs FastSurfer with the flags of +`run_fastsurfer.sh`: ```bash curl -fsSLO {{ LAUNCHER_URL }} diff --git a/doc/overview/install/MACOS.md b/doc/overview/install/MACOS.md index 9a6b2e5e4..514e5e654 100644 --- a/doc/overview/install/MACOS.md +++ b/doc/overview/install/MACOS.md @@ -160,13 +160,16 @@ Docker (Intel Macs) The package does not run on Intel Macs, but Docker does, with the full pipeline on the CPU (2 to 4 times slower than on Apple silicon). -1. Install [Docker Desktop for Mac](https://docs.docker.com/get-docker/), start it, and under - **Settings > Resources** set the memory to 15 GB (or the most you have; with less than that, it may fail). -2. Download the CPU image. Open a Terminal window and run: +```{include} setup/macos_intel.md +:relative-docs: . +``` + +Then run FastSurfer with the [launcher](../CONTAINERS.md#the-launcher), as in the [quick start](../QUICKSTART.md). +Without it, download the CPU image: - ```bash - docker pull deepmi/fastsurfer:cpu-v{{ FASTSURFER_VERSION }} - ``` +```bash +docker pull deepmi/fastsurfer:cpu-v{{ FASTSURFER_VERSION }} +``` -3. Run FastSurfer as in [Example 2](../EXAMPLES.md#example-2-fastsurfer-docker), with the CPU image and without - `--gpus all`. +and run FastSurfer as in [Example 2](../EXAMPLES.md#example-2-fastsurfer-docker), with the CPU image and without +`--gpus all`. diff --git a/doc/overview/install/WINDOWS.md b/doc/overview/install/WINDOWS.md index 9d565d413..d27f2ccaf 100644 --- a/doc/overview/install/WINDOWS.md +++ b/doc/overview/install/WINDOWS.md @@ -5,16 +5,16 @@ On Windows, run FastSurfer in our Docker image, with Docker Desktop on WSL2. The [FreeSurfer license](../INSTALL.md#freesurfer-license), which is free but not included: get it from FreeSurfer before your first full run. -Make sure that these are installed and running: -* [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) -* [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/) +Set up +------ +```{include} setup/windows.md +:relative-docs: . +``` For a GPU, the requirements in the NVIDIA GPU tab below apply as well. -The [launcher](../CONTAINERS.md#the-launcher) writes the Docker command for you. Run it in the terminal of your -WSL2 Linux distribution (for example Ubuntu, from the Start menu), not in PowerShell. Docker Desktop's `docker` -command is available there once you switch on the WSL integration for that distribution (Docker Desktop: -**Settings > Resources > WSL integration**). For example: +Then the [launcher](../CONTAINERS.md#the-launcher) writes the Docker command for you. In the Ubuntu terminal, for +example: ```bash curl -fsSLO {{ LAUNCHER_URL }} @@ -27,7 +27,9 @@ It picks the image for your NVIDIA GPU (or the CPU image), downloads it, shares runs it as your WSL user. Keep the data in your WSL home folder: Windows folders are available below `/mnt/c/`, but reading them from WSL is slower. -Without the launcher, choose the image for your hardware and run the commands in Windows PowerShell. +Without the launcher +-------------------- +Choose the image for your hardware and run the commands in Windows PowerShell. `````{tab-set} diff --git a/doc/overview/install/setup/linux.md b/doc/overview/install/setup/linux.md new file mode 100644 index 000000000..6423f93f5 --- /dev/null +++ b/doc/overview/install/setup/linux.md @@ -0,0 +1,19 @@ +FastSurfer runs in a container, with Apptainer (or its predecessor Singularity) or Docker. Check whether one of them +works, for example: + +```bash +# Apptainer (with Singularity: singularity --version) +apptainer --version +# Docker +docker run --rm hello-world +``` + +Clusters usually provide Apptainer, often as a module (`module load apptainer`, see your cluster's documentation). +Installing Apptainer or Docker needs administrator rights, so if neither works, ask your administrator to install +[Apptainer](https://apptainer.org/docs/admin/main/installation.html) or +[Docker](https://docs.docker.com/engine/install/). Docker also needs: + +- your user in the `docker` group, to run containers without `sudo` (log out and back in once you were added), +- for NVIDIA GPUs, the + [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + Apptainer needs only the NVIDIA driver. diff --git a/doc/overview/install/setup/macos_intel.md b/doc/overview/install/setup/macos_intel.md new file mode 100644 index 000000000..d148bac87 --- /dev/null +++ b/doc/overview/install/setup/macos_intel.md @@ -0,0 +1,7 @@ +1. Install [Docker Desktop for Mac](https://docs.docker.com/get-docker/), start it, and under **Settings > Resources** + set the memory to 15 GB (or the most you have; with less than that, it may fail). +2. Check Docker in a Terminal window, for example: + + ```bash + docker run --rm hello-world + ``` diff --git a/doc/overview/install/setup/windows.md b/doc/overview/install/setup/windows.md new file mode 100644 index 000000000..a788f26d5 --- /dev/null +++ b/doc/overview/install/setup/windows.md @@ -0,0 +1,16 @@ +FastSurfer runs in Docker, inside WSL2, the Linux environment of Windows: + +1. Install WSL2: open PowerShell as administrator, run `wsl --install`, and restart Windows. This also installs + Ubuntu. +2. Install [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/), start it, and switch on the + WSL integration for Ubuntu under **Settings > Resources > WSL integration**. +3. For an NVIDIA GPU, install the latest [NVIDIA driver for WSL](https://developer.nvidia.com/cuda/wsl) in Windows. +4. Open **Ubuntu** from the Start menu. Run the FastSurfer commands in this terminal, not in PowerShell. Check + Docker there, for example: + + ```bash + docker run --rm hello-world + ``` + +WSL2 gets only part of the computer's memory; the [system requirements](../../intro.rst#system-requirements) say how +much FastSurfer needs, and the launcher warns if Docker has less. diff --git a/test/utils/test_fastsurfer_launcher.py b/test/utils/test_fastsurfer_launcher.py index 24af4ff94..bb894292e 100644 --- a/test/utils/test_fastsurfer_launcher.py +++ b/test/utils/test_fastsurfer_launcher.py @@ -564,6 +564,26 @@ def test_plan_amd_build(): launcher.plan_amd_build(["--device", "cuda"], [old], False) +@pytest.mark.parametrize( + ("error", "system", "wsl", "advice"), + [ + # the errors Docker prints, and what the launcher tells the user to do instead + ("permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: " + "Get \"http://%2Fvar%2Frun%2Fdocker.sock/v1.45/info\": dial unix /var/run/docker.sock: connect: " + "permission denied", "Linux", False, "docker group"), + ("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?", + "Linux", False, "administrator starts it"), + ("Cannot connect to the Docker daemon at unix:///Users/me/.docker/run/docker.sock. Is the docker daemon " + "running?", "Darwin", False, "start Docker Desktop"), + ("The command 'docker' could not be found in this WSL 2 distro.\nWe recommend to activate the WSL " + "integration in Docker Desktop settings.", "Linux", True, "WSL integration"), + ("error during connect: something else\n", "Linux", False, "does not work: error during connect"), + ], +) +def test_docker_problem(error, system, wsl, advice): + assert advice in launcher.docker_problem(error, system, wsl) + + def test_plan_image_build_custom(): nvidia, amd = [gpu((8, 6))], [launcher.AmdGpu(0, (11, 0, 0), 24)] # an own image on a machine with only AMD GPUs gets them, whatever its name diff --git a/tools/fastsurfer b/tools/fastsurfer index bfff4ed6f..107796a0b 100755 --- a/tools/fastsurfer +++ b/tools/fastsurfer @@ -54,6 +54,8 @@ from pathlib import Path DOCKER_REPO = "deepmi/fastsurfer" DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/CONTAINERS.html" +# what to install on each system, for the messages about a missing or unusable container runtime +SETUP_URL = "https://deep-mi.org/FastSurfer/stable/overview/QUICKSTART.html#set-up-your-computer" LAUNCHER_URL = "https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer" # seconds to wait for Docker Hub and GitHub, so a machine without internet is not held up HUB_TIMEOUT = 5 @@ -1291,17 +1293,60 @@ def is_apple_silicon() -> bool: return out.stdout.strip() == "1" -def docker_memory_gib() -> float: - """The memory Docker's containers can use; on macOS and Windows, that of Docker's VM.""" +def is_wsl() -> bool: + """Whether this is Linux in WSL2 on Windows, whose kernel names Microsoft.""" + return platform.system() == "Linux" and "microsoft" in platform.release().lower() + + +def no_runtime_message(system: str, wsl: bool) -> str: + """What to install when neither Apptainer, Singularity nor Docker is, on this system.""" + if system == "Darwin": + return f"Docker is not installed. On an Intel Mac, FastSurfer runs in Docker Desktop, see {SETUP_URL}." + if wsl: + return (f"Docker is not available in this WSL distribution. Install Docker Desktop in Windows and switch on " + f"its WSL integration for this distribution (Settings > Resources > WSL integration), see {SETUP_URL}.") + return (f"Neither Apptainer, Singularity nor Docker is installed. On a cluster, Apptainer is often a module " + f"(module load apptainer); otherwise an administrator installs Apptainer or Docker, see {SETUP_URL}.") + + +def docker_problem(error: str, system: str, wsl: bool) -> str: + """What to do when `docker info` fails with this error, on this system.""" + lowered = error.lower() + desktop = system == "Darwin" or wsl + if "could not be found in this wsl" in lowered: + return (f"Docker Desktop's WSL integration is off for this distribution: switch it on under Settings > " + f"Resources > WSL integration, see {SETUP_URL}.") + if "permission denied" in lowered and not desktop: + return (f"Docker is installed, but this user may not use it: an administrator adds you to the docker group, " + f"then log out and back in, see {SETUP_URL}.") + if "cannot connect to the docker daemon" in lowered or "is the docker daemon running" in lowered: + if desktop: + return "Docker is installed, but not running: start Docker Desktop." + return "Docker is installed, but its service is not running: an administrator starts it." + reason = (error.strip().splitlines() or ["docker info failed"])[-1] + return f"Docker is installed, but does not work: {reason}" + + +def docker_info() -> tuple[float, set[str]]: + """ + The memory Docker's containers can use (on macOS and Windows, that of Docker's VM) and the runtimes Docker + knows, e.g. nvidia with the NVIDIA Container Toolkit; stops with what to do if Docker cannot be used. + """ + import json + try: - out = subprocess.run(["docker", "info", "--format", "{{.MemTotal}}"], + out = subprocess.run(["docker", "info", "--format", "{{.MemTotal}} {{json .Runtimes}}"], capture_output=True, text=True, timeout=60) except (OSError, subprocess.SubprocessError) as error: raise LauncherError(f"Could not ask Docker for its settings: {error}") from error if out.returncode != 0: - reason = (out.stderr.strip().splitlines() or ["docker info failed"])[-1] - raise LauncherError(f"Docker is installed, but not running: {reason}") - return int(out.stdout.strip()) / 2**30 + raise LauncherError(docker_problem(out.stderr or out.stdout, platform.system(), is_wsl())) + memory, _, runtimes = out.stdout.strip().partition(" ") + try: + names = set(json.loads(runtimes or "{}")) + except ValueError: + names = set() + return int(memory) / 2**30, names def _last_value(args: list[str], flag: str) -> str | None: @@ -1506,7 +1551,7 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b download_only = options.update and not other and not fastsurfer_args installed = {name for name in ("apptainer", "singularity", "docker") if shutil.which(name)} if not installed: - raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") + raise LauncherError(no_runtime_message(platform.system(), is_wsl())) if options.runtime != "auto" and options.runtime not in installed: raise LauncherError(f"{options.runtime} is not installed.") if brun and not {"--subjects", "--subjects_list", "--subject_list", "--help"} & set(fastsurfer_args): @@ -1607,13 +1652,13 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, build.devices, workdir, interactive, build.amd, build.env, amd_device_groups() if build.amd else None, tty)) - if not options.print_only and not download_only: - memory = docker_memory_gib() - if memory < MIN_DOCKER_MEMORY_GIB: + if not options.print_only: + memory, runtimes = docker_info() + if memory < MIN_DOCKER_MEMORY_GIB and not download_only: if platform.system() == "Darwin": where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " "--memory 16)") - elif "microsoft" in platform.release().lower(): + elif is_wsl(): # Docker Desktop on WSL2 runs in the memory WSL2 has where = ("of WSL2 in %UserProfile%\\.wslconfig (memory=16GB in the [wsl2] section), then run " "wsl --shutdown") @@ -1621,6 +1666,14 @@ def container_commands(options: argparse.Namespace, script: str | None, other: b where = "for Docker" info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " f"15 GB. Increase the memory {where}.", level="WARNING") + # Docker on Linux reaches NVIDIA GPUs through the NVIDIA Container Toolkit, which Docker Desktop has + # built in; it registers the nvidia runtime or, in newer versions, only installs its hook + toolkit = "nvidia" in runtimes or any(shutil.which(name) for name in ("nvidia-container-runtime-hook", + "nvidia-ctk")) + if build.gpu and not build.amd and platform.system() == "Linux" and not is_wsl() and not toolkit: + info(f"Docker seems to lack the NVIDIA Container Toolkit, which it needs for the GPU. If Docker " + f"stops with 'could not select device driver', an administrator installs it (see {SETUP_URL}), " + f"or --device cpu runs FastSurfer on the CPU.", level="WARNING") else: if kind == "sif" and not Path(plan.image).is_file(): raise LauncherError(f"--image {value}: no such file.")