diff --git a/README.md b/README.md index 331775fb0..d5aef93ca 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,15 @@ The images we provide on [Docker Hub](https://hub.docker.com/r/deepmi/fastsurfer ### Usage All installation methods use the `run_fastsurfer.sh` call interface (replace the placeholder `` with [FastSurfer flags](doc/scripts/RUN_FASTSURFER.md#required-arguments)), which is the general starting point for FastSurfer. However, there are different ways to call this script depending on the installation, which we explain here: -1. For container installations, you need to set up the container (`` or ``) in addition to the ``: +1. For container installations, the [launcher](doc/overview/CONTAINERS.md#the-launcher) is the easiest way: it takes the `` and writes and runs the container command, with the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU): + + ```bash + curl -fsSLO https://deep-mi.org/FastSurfer/stable/fastsurfer + chmod +x fastsurfer + ./fastsurfer + ``` + + Written by hand, the container command needs the container's setup (`` or ``) in addition to the ``: 1. For __Singularity__, the syntax is ```text diff --git a/doc/CONVENTIONS.md b/doc/CONVENTIONS.md index a330b8ed6..88da68214 100644 --- a/doc/CONVENTIONS.md +++ b/doc/CONVENTIONS.md @@ -129,6 +129,13 @@ Names are snake_case, except `subjectX` and the file names of container images. example `-v C:/Users/user/my_mri_data:/home/user/my_mri_data`. The license file is mounted from its made-up path: `-v C:/path/to/your/freesurfer/license_file:/home/user/freesurfer_license`. +## Shared setup + +What each system needs before FastSurfer runs (a container runtime, WSL2, Docker Desktop) is written once, in +`doc/overview/install/setup/`, and included in the quick start and the install page of that system with +`:relative-docs: .`. Links in these files are relative to that folder. `exclude_patterns` in `doc/conf.py` keeps them +from becoming pages of their own. + ## Line length Lines in code fences in the documentation are at most 80 characters, so there is no scroll bar. Count the rendered @@ -157,7 +164,9 @@ supported versions. `doc/conf.py`), with a separate text for `{{ FASTSURFER_VERSION }}`, `{{ CUDA_STRING }}` and both: that the commands use the latest release (in the development documentation), and that images for other CUDA versions are available on Docker Hub. Do not add such notes by hand. Where the version is not about an image, such as the folder of the macOS -package, use `{{ PACKAGE_VERSION }}`, the same version without the note. +package, use `{{ PACKAGE_VERSION }}`, the same version without the note. The download of the launcher uses +`{{ LAUNCHER_URL }}`, the copy published next to the pages of the same tree (`stable` or `dev`). `README.md` is +shown on GitHub, where nothing is substituted, so it names the `stable` copy. What this tree builds comes from this tree, not from that release: `{{ CUDA_DEFAULT_STRING }}` and `{{ ROCM_DEFAULT_STRING }}` (with `{{ CUDA_DEFAULT_VERSION }}` and `{{ ROCM_DEFAULT_VERSION }}`) are the default CUDA diff --git a/doc/conf.py b/doc/conf.py index 6fd5b186a..3996eceb1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -209,6 +209,8 @@ def _tool_values_gitref(ref: str | None, *keys: str) -> tuple[str, ...]: "PYTHON_VERSION": version_python, "UBUNTU_VERSION": version_ubuntu, "FREESURFER_VERSION": version_freesurfer, + # the launcher published next to this tree's pages (html_extra_path), so each tree links to its own copy + "LAUNCHER_URL": f"https://deep-mi.org/FastSurfer/{'stable' if documents_a_release else 'dev'}/fastsurfer", } templates_path = ["_templates"] @@ -220,6 +222,8 @@ def _tool_values_gitref(ref: str | None, *keys: str) -> tuple[str, ...]: # instructions for writing the documentation, not part of it "AGENTS.md", "CONVENTIONS.md", + # the setup of each system, included in the quick start and the install pages rather than pages of their own + "overview/install/setup/*", ] @@ -239,6 +243,8 @@ def _tool_values_gitref(ref: str | None, *keys: str) -> tuple[str, ...]: # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "furo" html_static_path = ["_static"] +# served next to the pages, so the launcher downloads from deep-mi.org/FastSurfer// +html_extra_path = ["../tools/fastsurfer"] html_js_files = ["doc-version-link.js"] # points the announcement bar at the sibling doc tree html_title = project html_show_sphinx = False diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 4738085ca..3c1d7f5b8 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -21,13 +21,15 @@ as a "simulated computer inside the computer". The launcher ------------ -`fastsurfer_container.py` writes and runs the container command for you. It takes the same flags as +The launcher `fastsurfer` writes and runs the container command for you. It takes the same flags as `run_fastsurfer.sh` and needs only Python 3.8 or newer, not FastSurfer itself. It - uses Apptainer, Singularity or Docker, whichever is installed (choose one with `--runtime`), - picks the image that runs on your NVIDIA GPU and driver, or the CPU image without one, see [which image fits your GPU](install/LINUX.md#nvidia-gpus); with several GPUs and no `--device cuda:`, it uses the newest one with enough memory and says which, +- runs the [ROCm image](install/LINUX.md#amd-gpus-experimental) on an AMD GPU (experimental) if there + is no NVIDIA GPU, or with `--image rocm`, - downloads the image once (for Apptainer, into `$HOME/my_singularity_images`), - shares the files and directories the flags name with the container, at the same paths, so relative paths work too, @@ -37,8 +39,9 @@ The launcher Download it, and run FastSurfer: ```bash -curl -fsSLO https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer_container.py -python3 fastsurfer_container.py \ +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer +./fastsurfer \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ --sid subjectX --sd $HOME/my_fastsurfer_analysis \ --3T --threads 4 @@ -59,18 +62,48 @@ version, it says how many releases it is behind. A GPU that the newest images no them on the CPU. The image of an older release may still support that GPU, but the launcher recommends the newest release on the CPU, or a machine with a newer GPU, over an old version. -With `--brun`, it runs `brun_fastsurfer.sh` for several subjects; the paths in a subjects list have -to be absolute. `--print_only` prints the commands instead of running them, `--update_launcher` +`--exec ` runs another program of the image instead of `run_fastsurfer.sh`. It comes last, as +all arguments after the program go to the program. The other FastSurfer scripts take FastSurfer's flags +and are handled like `run_fastsurfer.sh`: + +```bash +# several subjects; the paths in a subjects list have to be absolute +./fastsurfer --exec brun_fastsurfer.sh \ + --subjects_list $HOME/my_mri_data/subjects_list.txt \ + --sd $HOME/my_fastsurfer_analysis +# the time points of one person (longitudinal processing) +./fastsurfer --exec long_fastsurfer.sh --tid subjectX \ + --t1s $HOME/my_mri_data/subjectX/t1_weighted_20210315.nii.gz \ + $HOME/my_mri_data/subjectX/t1_weighted_20230920.nii.gz \ + --tpids subjectX_20210315 subjectX_20230920 \ + --sd $HOME/my_fastsurfer_analysis +``` + +([long_fastsurfer.sh](../scripts/long_fastsurfer.rst) explains longitudinal processing.) Any other +program runs in the current directory, for example to convert a segmentation to NIfTI: + +```bash +./fastsurfer --exec nib-convert \ + subjectX/mri/aparc.DKTatlas+aseg.deep.mgz aparc.nii.gz +``` + +The launcher shares the current directory with the container, as it does the directories of absolute +paths in the arguments, so relative paths only work inside the current directory. If several FastSurfer +images are installed, such a program uses the newest instead of asking. + +`--print_only` prints the commands instead of running them, `--update_launcher` replaces the launcher with the current one, and `--help` lists all launcher options. The sections below explain the commands it runs. Before starting many runs at once, e.g. a SLURM array, run the launcher once (e.g. on one subject), so the image is downloaded and built only once rather than by every job. -On a Mac with Apple silicon, use the [macOS package](install/MACOS.md) instead: the images are built for -Intel and AMD processors, which such a Mac only emulates, much more slowly and without its GPU. The -launcher therefore stops there and points to the package, unless you pass `--runtime docker`. With -Docker, it also warns if Docker may use less memory than FastSurfer needs. +On a Mac with Apple silicon, the launcher runs the installed [macOS package](install/MACOS.md) instead, +with the same flags and `--exec`: the images are built for Intel and AMD processors, which such a Mac +only emulates, much more slowly and without its GPU. It chooses among installed package versions as +among images, and `--image 2.6.0` keeps a study on one version. Without a package, `--update` +downloads it and opens the macOS Installer. To run an image in Docker anyway, pass `--runtime docker`. +With Docker, the launcher also warns if Docker may use less memory than FastSurfer needs. Using Apptainer (or Singularity) -------------------------------- diff --git a/doc/overview/INSTALL.md b/doc/overview/INSTALL.md index d51b2fbff..661bd1a86 100644 --- a/doc/overview/INSTALL.md +++ b/doc/overview/INSTALL.md @@ -1,7 +1,8 @@ Installation ============ -FastSurfer works the same way on every system: you call `run_fastsurfer.sh` with a T1-weighted MRI image. How you -install it depends on your system. Pick yours: +FastSurfer works the same way on every system: you call `run_fastsurfer.sh` with a T1-weighted MRI image. To try it, +the [quick start](QUICKSTART.md#set-up-your-computer) sets up your computer and runs FastSurfer with the launcher in +a few steps. How you install it for your own work depends on your system. Pick yours: ````{grid} 1 2 2 2 :gutter: 3 diff --git a/doc/overview/QUICKSTART.md b/doc/overview/QUICKSTART.md index 9b5752fb2..1705705fc 100644 --- a/doc/overview/QUICKSTART.md +++ b/doc/overview/QUICKSTART.md @@ -1,114 +1,109 @@ Quick Start =========== -This example segments one brain MRI in a few minutes. Choose how you run FastSurfer: in a container on Linux -(Apptainer, Singularity or Docker), with the [macOS package](install/MACOS.md), or without installing anything in -[Google Colab](#google-colab). Other systems are covered in the [installation guide](INSTALL.md). +This example segments one brain MRI in a few minutes with `fastsurfer`, a launcher that runs FastSurfer with the +same commands on Linux, Windows and macOS. It takes the flags of `run_fastsurfer.sh` and sets up the rest: on Linux, +Windows and Intel Macs, it picks, downloads and runs the right container image; on a Mac with Apple silicon, it runs +the installed macOS package. You can also try FastSurfer without installing anything in +[Google Colab](#google-colab). -Segment an example image ------------------------- -With a GPU (8 GB of graphics memory) this takes well under a minute; without one, FastSurfer uses the CPU and takes -a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 mm voxel sizes, no preprocessing, -`.nii`, `.nii.gz` or `.mgz`), or the example image the commands download. +The launcher is meant for getting started and for single images. For larger projects, such as whole datasets or +runs on an HPC cluster, we recommend running FastSurfer directly, which gives full control over images, paths and +resources: see the [installation guide](INSTALL.md), [Running FastSurfer in a container](CONTAINERS.md), and the +scripts for [many subjects](../scripts/BATCH.md) and for [SLURM clusters](../scripts/SLURM.md). + +Set up your computer +-------------------- +Once, set up what the launcher runs FastSurfer with: `````{tab-set} -````{tab-item} Apptainer / Singularity -```bash -# 0. Create a directory for this test -mkdir fastsurfer_test -cd fastsurfer_test +````{tab-item} Linux +:sync: linux -# 1. Build the Apptainer image from our Docker image (only the first time) -mkdir -p $HOME/my_singularity_images -singularity build \ - $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ - docker://deepmi/fastsurfer:{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }} +```{include} install/setup/linux.md +:relative-docs: . +``` -# 2. Download an example brain MRI (if you don't have your own) -# If you have your own, copy it to this directory and adjust -# the filename after --t1 below. -curl -k \ - https://surfer.nmr.mgh.harvard.edu/pub/data/tutorial_data/buckner_data/tutorial_subjs/140/mri/orig.mgz \ - -o "./140_orig.mgz" +The launcher picks the image for your NVIDIA GPU (or AMD GPU, experimental, or the CPU) and downloads it on the +first run. [Linux](install/LINUX.md) explains the images. +```` -# 3. Run FastSurfer (full brain segmentation only) -singularity exec --nv \ - --no-mount home,cwd -e \ - -B "$PWD" \ - $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ - /fastsurfer/run_fastsurfer.sh \ - --t1 "$PWD/140_orig.mgz" \ - --sid subjectX --sd "$PWD" \ - --seg_only --no_biasfield --no_cereb --no_hypothal +````{tab-item} Windows +:sync: windows + +```{include} install/setup/windows.md +:relative-docs: . ``` + +[Windows](install/WINDOWS.md) has more details. ```` -````{tab-item} Docker -```bash -# 0. Create a directory for this test -mkdir fastsurfer_test -cd fastsurfer_test +````{tab-item} Mac with Apple silicon +:sync: macos -# 1. Download an example brain MRI (if you don't have your own) -# If you have your own, copy it to this directory and adjust -# the filename after --t1 below. -curl -k \ - https://surfer.nmr.mgh.harvard.edu/pub/data/tutorial_data/buckner_data/tutorial_subjs/140/mri/orig.mgz \ - -o "./140_orig.mgz" +FastSurfer runs natively with the [macOS package](install/MACOS.md), which uses the Mac's GPU (M1 or newer). +Download and install it with the button on that page, or let the launcher do it: after step 1 below, +`./fastsurfer --update` downloads the package and opens the macOS Installer. macOS may block the installer, because +it is not signed by Apple yet; [Install](install/MACOS.md#install) explains how to allow it. +```` + +````{tab-item} Intel Mac +:sync: macos-intel + +FastSurfer runs in Docker, on the CPU (the macOS package needs Apple silicon): -# 2. Run FastSurfer (full brain segmentation only) -docker run --gpus all -v "$PWD:$PWD" \ - --rm --user $(id -u):$(id -g) \ - deepmi/fastsurfer:{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }} \ - --t1 "$PWD/140_orig.mgz" \ - --sid subjectX --sd "$PWD" \ - --seg_only --no_biasfield --no_cereb --no_hypothal +```{include} install/setup/macos_intel.md +:relative-docs: . ``` ```` -````{tab-item} macOS package -[Install the package](install/MACOS.md), open the FastSurfer app from your Applications folder, and run these -commands in the Terminal window it opens: +````` + +If the launcher still cannot run FastSurfer, it says what is missing and how to fix it. + +Segment an example image +------------------------ +With a GPU (8 GB of graphics memory) this takes well under a minute; without one, FastSurfer uses the CPU and takes +a few minutes longer. You can use your own T1-weighted full head MRI (0.7 to 1 mm voxel sizes, no preprocessing, +`.nii`, `.nii.gz` or `.mgz`), or the example image the commands download. On Windows, run the commands in the +Ubuntu terminal. ```bash # 0. Create a directory for this test mkdir fastsurfer_test cd fastsurfer_test -# 1. Download an example brain MRI (if you don't have your own) +# 1. Download the launcher (only the first time) +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer + +# 2. Download an example brain MRI (if you don't have your own) # If you have your own, copy it to this directory and adjust # the filename after --t1 below. curl -k \ https://surfer.nmr.mgh.harvard.edu/pub/data/tutorial_data/buckner_data/tutorial_subjs/140/mri/orig.mgz \ -o "./140_orig.mgz" -# 2. Run FastSurfer (full brain segmentation only) -run_fastsurfer.sh --t1 "$PWD/140_orig.mgz" \ - --sid subjectX --sd "$PWD" \ +# 3. Run FastSurfer (full brain segmentation only) +./fastsurfer --t1 140_orig.mgz --sid subjectX --sd . \ --seg_only --no_biasfield --no_cereb --no_hypothal ``` -```` - -````` That's it, it will run the full brain segmentation. For speed, we switched off the cerebellum and hypothalamic sub-segmentation (would add a couple minutes). We also switched off the bias field correction, which is used to compute partial volume estimates for the statsfiles, so you might want to switch it on again if you want the volume statistics text file (under ```subjectX/stats```). Also if you need the estimated total intracranial volume for correcting the stats, you would either need to run the surface stream or switch on the Talairach registration with -`--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license) and pass it into the container, see [Running FastSurfer in a container](CONTAINERS.md). +`--tal_reg` in the segmentation module. For the full surface stream, just remove the ```--seg_only```; you then need a [FreeSurfer license file](INSTALL.md#freesurfer-license): pass it with `--fs_license `, or set the environment variable `FS_LICENSE` to its path, which the launcher passes on. -You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer, for example with the Singularity image (in the macOS console, call `nib-convert` directly): +You will find the full brain segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.mgz``` in FreeSurfer's MGZ file format. To convert it back to nifti (if you prefer), run `nib-convert`, which comes with FastSurfer: ```bash # Convert mgz to nifti -singularity exec --nv \ - --no-mount home -e \ - -B "$PWD" \ - $HOME/my_singularity_images/fastsurfer-{{ CUDA_STRING }}-v{{ FASTSURFER_VERSION }}.sif \ - nib-convert "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.mgz" \ - "$PWD/subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz" +./fastsurfer --exec nib-convert \ + subjectX/mri/aparc.DKTatlas+aseg.deep.mgz \ + subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz ``` -and find the segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz```. If you have FreeSurfer installed, just use FreeView to look at the result (or really any other image viewer): +Then find the segmentation in ```./subjectX/mri/aparc.DKTatlas+aseg.deep.nii.gz```. If you have FreeSurfer installed, just use FreeView to look at the result (or really any other image viewer): ```bash # FreeView diff --git a/doc/overview/install/LINUX.md b/doc/overview/install/LINUX.md index 9fe57d288..9641d3aae 100644 --- a/doc/overview/install/LINUX.md +++ b/doc/overview/install/LINUX.md @@ -9,6 +9,25 @@ and for the CPU only, on [Docker Hub](https://hub.docker.com/r/deepmi/fastsurfer A GPU with enough memory makes the segmentation much faster, see the [system requirements](../intro.rst#system-requirements). Without a GPU, use the [CPU image](#cpu-only). +Set up +------ +```{include} setup/linux.md +:relative-docs: . +``` + +Then the [launcher](../CONTAINERS.md#the-launcher) does the rest: it picks the image for your NVIDIA GPU and driver +(or for an AMD GPU, experimental, or the CPU image), downloads it, and runs FastSurfer with the flags of +`run_fastsurfer.sh`: + +```bash +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer +./fastsurfer --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ + --sid subjectX --sd $HOME/my_fastsurfer_analysis +``` + +The sections below explain the images and how to download them by hand. + The images are tagged `deepmi/fastsurfer:-v`, where `` is `cu` for NVIDIA GPUs (see [below](#nvidia-gpus)), `rocm` for AMD GPUs (experimental), or `cpu` without GPU support (smaller and thus faster to download), and `` is the FastSurfer version. `latest` points to the newest NVIDIA image, @@ -110,7 +129,12 @@ semi-supported) GPU and the right kernel version. Install the AMD kernel modules [ROCm installation instructions](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/), and add your user to the groups they name. -Then download our image for ROCm {{ ROCM_VERSION }}: +The [launcher](../CONTAINERS.md#the-launcher) uses the ROCm image when the machine has an AMD GPU and no NVIDIA GPU, +or when you pass `--image rocm`. It runs with Docker or Apptainer and adds the flags below. Its image supports +several AMD architectures; for another GPU of the same family, such as the W6600 (gfx1032, family gfx1030), it sets +`HSA_OVERRIDE_GFX_VERSION` and says so. + +Without the launcher, download our image for ROCm {{ ROCM_VERSION }}: ```bash docker pull deepmi/fastsurfer:{{ ROCM_STRING }}-v{{ FASTSURFER_VERSION }} diff --git a/doc/overview/install/MACOS.md b/doc/overview/install/MACOS.md index 61a384156..514e5e654 100644 --- a/doc/overview/install/MACOS.md +++ b/doc/overview/install/MACOS.md @@ -72,6 +72,19 @@ run_fastsurfer.sh --sd $HOME/my_fastsurfer_analysis --sid subjectX \ FastSurfer uses the Apple GPU (`mps`) automatically, so no `--device` flag is needed. +Without opening the console, the [launcher](../CONTAINERS.md#the-launcher) `fastsurfer` runs the installed package +from any Terminal window, with the same flags and the same commands as on Linux: + +```bash +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer +./fastsurfer --seg_only --sd $HOME/my_fastsurfer_analysis --sid subjectX \ + --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz +``` + +If several versions are installed, it asks for one, e.g. `--image 2.6.0`. `./fastsurfer --update` downloads the +newest package and opens the installer. + FreeSurfer license ------------------ On macOS, FastSurfer cannot find the [FreeSurfer license](../INSTALL.md#freesurfer-license) inside the bundled @@ -147,13 +160,16 @@ Docker (Intel Macs) The package does not run on Intel Macs, but Docker does, with the full pipeline on the CPU (2 to 4 times slower than on Apple silicon). -1. Install [Docker Desktop for Mac](https://docs.docker.com/get-docker/), start it, and under - **Settings > Resources** set the memory to 15 GB (or the most you have; with less than that, it may fail). -2. Download the CPU image. Open a Terminal window and run: +```{include} setup/macos_intel.md +:relative-docs: . +``` + +Then run FastSurfer with the [launcher](../CONTAINERS.md#the-launcher), as in the [quick start](../QUICKSTART.md). +Without it, download the CPU image: - ```bash - docker pull deepmi/fastsurfer:cpu-v{{ FASTSURFER_VERSION }} - ``` +```bash +docker pull deepmi/fastsurfer:cpu-v{{ FASTSURFER_VERSION }} +``` -3. Run FastSurfer as in [Example 2](../EXAMPLES.md#example-2-fastsurfer-docker), with the CPU image and without - `--gpus all`. +and run FastSurfer as in [Example 2](../EXAMPLES.md#example-2-fastsurfer-docker), with the CPU image and without +`--gpus all`. diff --git a/doc/overview/install/WINDOWS.md b/doc/overview/install/WINDOWS.md index 5061d2c46..d27f2ccaf 100644 --- a/doc/overview/install/WINDOWS.md +++ b/doc/overview/install/WINDOWS.md @@ -5,11 +5,31 @@ On Windows, run FastSurfer in our Docker image, with Docker Desktop on WSL2. The [FreeSurfer license](../INSTALL.md#freesurfer-license), which is free but not included: get it from FreeSurfer before your first full run. -Make sure that these are installed and running: -* [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) -* [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/) +Set up +------ +```{include} setup/windows.md +:relative-docs: . +``` + +For a GPU, the requirements in the NVIDIA GPU tab below apply as well. + +Then the [launcher](../CONTAINERS.md#the-launcher) writes the Docker command for you. In the Ubuntu terminal, for +example: + +```bash +curl -fsSLO {{ LAUNCHER_URL }} +chmod +x fastsurfer +./fastsurfer --seg_only --sd $HOME/my_fastsurfer_analysis --sid subjectX \ + --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz +``` + +It picks the image for your NVIDIA GPU (or the CPU image), downloads it, shares the folders with the container and +runs it as your WSL user. Keep the data in your WSL home folder: Windows folders are available below `/mnt/c/`, but +reading them from WSL is slower. -Then choose the image for your hardware. Run the commands in Windows PowerShell. +Without the launcher +-------------------- +Choose the image for your hardware and run the commands in Windows PowerShell. `````{tab-set} @@ -79,3 +99,4 @@ Linux user IDs anyway, so pass a fixed user ID instead, for example `1000:1000`. Docker Desktop reserves only part of your memory for WSL2. FastSurfer needs at least the memory in the [system requirements](../intro.rst#system-requirements); if it fails, check [how much memory your WSL2 distribution has](https://www.aleksandrhovhannisyan.com/blog/limiting-memory-usage-in-wsl-2/). +The launcher warns if Docker has less, and the limit is set in `%UserProfile%\.wslconfig`. diff --git a/doc/overview/install/setup/linux.md b/doc/overview/install/setup/linux.md new file mode 100644 index 000000000..6423f93f5 --- /dev/null +++ b/doc/overview/install/setup/linux.md @@ -0,0 +1,19 @@ +FastSurfer runs in a container, with Apptainer (or its predecessor Singularity) or Docker. Check whether one of them +works, for example: + +```bash +# Apptainer (with Singularity: singularity --version) +apptainer --version +# Docker +docker run --rm hello-world +``` + +Clusters usually provide Apptainer, often as a module (`module load apptainer`, see your cluster's documentation). +Installing Apptainer or Docker needs administrator rights, so if neither works, ask your administrator to install +[Apptainer](https://apptainer.org/docs/admin/main/installation.html) or +[Docker](https://docs.docker.com/engine/install/). Docker also needs: + +- your user in the `docker` group, to run containers without `sudo` (log out and back in once you were added), +- for NVIDIA GPUs, the + [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + Apptainer needs only the NVIDIA driver. diff --git a/doc/overview/install/setup/macos_intel.md b/doc/overview/install/setup/macos_intel.md new file mode 100644 index 000000000..d148bac87 --- /dev/null +++ b/doc/overview/install/setup/macos_intel.md @@ -0,0 +1,7 @@ +1. Install [Docker Desktop for Mac](https://docs.docker.com/get-docker/), start it, and under **Settings > Resources** + set the memory to 15 GB (or the most you have; with less than that, it may fail). +2. Check Docker in a Terminal window, for example: + + ```bash + docker run --rm hello-world + ``` diff --git a/doc/overview/install/setup/windows.md b/doc/overview/install/setup/windows.md new file mode 100644 index 000000000..a788f26d5 --- /dev/null +++ b/doc/overview/install/setup/windows.md @@ -0,0 +1,16 @@ +FastSurfer runs in Docker, inside WSL2, the Linux environment of Windows: + +1. Install WSL2: open PowerShell as administrator, run `wsl --install`, and restart Windows. This also installs + Ubuntu. +2. Install [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/), start it, and switch on the + WSL integration for Ubuntu under **Settings > Resources > WSL integration**. +3. For an NVIDIA GPU, install the latest [NVIDIA driver for WSL](https://developer.nvidia.com/cuda/wsl) in Windows. +4. Open **Ubuntu** from the Start menu. Run the FastSurfer commands in this terminal, not in PowerShell. Check + Docker there, for example: + + ```bash + docker run --rm hello-world + ``` + +WSL2 gets only part of the computer's memory; the [system requirements](../../intro.rst#system-requirements) say how +much FastSurfer needs, and the launcher warns if Docker has less. diff --git a/pyproject.toml b/pyproject.toml index 594aff9bd..20067df77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,9 +225,9 @@ add_ignore = 'D100,D104,D107' [tool.ruff] line-length = 120 target-version = "py312" -# recon_surf/fs_time is python, but carries no .py suffix because the pipeline invokes it as a -# command; without this ruff would skip it -extend-include = ["recon_surf/fs_time"] +# recon_surf/fs_time and the launcher tools/fastsurfer are python, but carry no .py suffix because they +# are invoked as commands; without this ruff would skip them +extend-include = ["recon_surf/fs_time", "tools/fastsurfer"] extend-exclude = [ "build", "checkpoints", diff --git a/test/lint/test_launcher_builds.py b/test/lint/test_launcher_builds.py index df44de20c..b4e58833c 100644 --- a/test/lint/test_launcher_builds.py +++ b/test/lint/test_launcher_builds.py @@ -13,13 +13,14 @@ # limitations under the License. """ -Guard the copy of the CUDA images in tools/fastsurfer_container.py. +Guard the copy of the CUDA images in the launcher, tools/fastsurfer. The launcher runs with any system Python and without FastSurfer installed, so it cannot import FastSurferCNN/gpu_support.py and keeps its own copy of the CUDA builds and their minimum drivers. A copy left behind when the CUDA versions change makes it pick an image that is not released. """ +import importlib.machinery import importlib.util import sys import tomllib @@ -29,15 +30,16 @@ def _load(name: str, path: Path): - # from the path, because the lint job installs nothing but pytest; registered in sys.modules, - # which dataclasses need - spec = importlib.util.spec_from_file_location(name, path) + # from the path, because the lint job installs nothing but pytest, with an explicit loader for the + # launcher, which has no .py suffix; registered in sys.modules, which dataclasses need + loader = importlib.machinery.SourceFileLoader(name, str(path)) + spec = importlib.util.spec_from_file_location(name, path, loader=loader) module = sys.modules.setdefault(name, importlib.util.module_from_spec(spec)) spec.loader.exec_module(module) return module -launcher = _load("fastsurfer_container", FASTSURFER_HOME / "tools" / "fastsurfer_container.py") +launcher = _load("fastsurfer_launcher_for_lint", FASTSURFER_HOME / "tools" / "fastsurfer") gpu_support = _load("gpu_support_for_lint", FASTSURFER_HOME / "FastSurferCNN" / "gpu_support.py") @@ -55,5 +57,8 @@ def test_launcher_capabilities_cover_builds(): def test_launcher_default_build_matches_pyproject(): with open(FASTSURFER_HOME / "pyproject.toml", "rb") as fp: - cuda = tomllib.load(fp)["tool"]["cuda"]["version"] - assert launcher.NEWEST_BUILD[0] == tuple(int(v) for v in cuda.split(".")) + tool = tomllib.load(fp)["tool"] + assert launcher.NEWEST_BUILD[0] == tuple(int(v) for v in tool["cuda"]["version"].split(".")) + # the ROCm image; a new version also needs its architectures (ROCM_ARCHS) read from its PyTorch + rocm = tool["rocm"]["version"] + assert launcher.ROCM_BUILD == (tuple(int(v) for v in rocm.split(".")), f"rocm{rocm}") diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_launcher.py similarity index 58% rename from test/utils/test_fastsurfer_container.py rename to test/utils/test_fastsurfer_launcher.py index 07539e205..bb894292e 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_launcher.py @@ -12,19 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test the image choice and the path handling of tools/fastsurfer_container.py.""" +"""Test the image choice and the path handling of the launcher, tools/fastsurfer.""" +import importlib.machinery import importlib.util import platform +import subprocess import sys from pathlib import Path import pytest -# a script rather than a module of FastSurfer, so it is loaded from its path; dataclasses need it -# registered in sys.modules -_SCRIPT = Path(__file__).parent.parent.parent / "tools" / "fastsurfer_container.py" -_spec = importlib.util.spec_from_file_location("fastsurfer_container", _SCRIPT) +# a command rather than a module of FastSurfer, without the .py suffix, so it is loaded from its path with +# an explicit loader; dataclasses need it registered in sys.modules +_SCRIPT = str(Path(__file__).parent.parent.parent / "tools" / "fastsurfer") +_spec = importlib.util.spec_from_file_location( + "fastsurfer_launcher", _SCRIPT, loader=importlib.machinery.SourceFileLoader("fastsurfer_launcher", _SCRIPT)) launcher = sys.modules.setdefault(_spec.name, importlib.util.module_from_spec(_spec)) _spec.loader.exec_module(launcher) @@ -123,7 +126,7 @@ def test_gpu_request(): assert launcher.gpu_request(["--device", "cuda:0-2,5"], brun=True) == ("cuda", [0, 1, 2, 5]) for value in ("cuda:0,1", "cuda:0-3"): for flag in ("--device", "--viewagg_device"): - with pytest.raises(launcher.LauncherError, match="--brun"): + with pytest.raises(launcher.LauncherError, match="brun_fastsurfer.sh"): launcher.gpu_request([flag, value]) assert launcher.gpu_request(["--device", "cpu"]) == ("cpu", None) # the view aggregation can need a GPU while the segmentation runs on the CPU @@ -294,6 +297,16 @@ def test_decide_image_several_installed(tmp_path): plan = decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}, update=True) assert plan.build_from is not None and plan.tag == "cu132-v2.6.0" assert any("docker image rm deepmi/fastsurfer:cu132-v2.5.4" in note for _, note in plan.notes) + # a runtime named with --runtime counts only its own images while it has one + assert decide(tmp_path, runtime="apptainer", docker={"cu132-v2.5.4": "sha256:b"}, + sif={"cu132-v2.6.0": ""}).tag == "cu132-v2.6.0" + assert decide(tmp_path, runtime="docker", docker={"cu132-v2.5.4": "sha256:b"}, + sif={"cu132-v2.6.0": ""}).tag == "cu132-v2.5.4" + # a program other than FastSurfer (--exec) takes the newest instead of stopping, without study advice + plan = launcher.decide_image("auto", "", "cu132", "auto", BOTH, {"cu132-v2.5.4": "sha256:b"}, {"cu132-v2.6.0": ""}, + HUB, tmp_path, False, other_program=True) + assert plan.tag == "cu132-v2.6.0" and any("uses the newest" in note for _, note in plan.notes) + assert not any("For a study" in note for _, note in plan.notes) def test_decide_image_digest(tmp_path): @@ -324,12 +337,17 @@ def test_decide_image_named(tmp_path): assert decide(tmp_path, kind="docker", value="my_fs:cu132", local_id="sha256:1").runtime == "docker" plan = decide(tmp_path, kind="docker", value="my_fs:cu132", runtime="apptainer", local_id="sha256:1") assert (plan.build_from, plan.record) == ("docker-daemon://my_fs:cu132", "sha256:1") - (tmp_path / "my_fs-cu132.sif").write_bytes(b"") - (tmp_path / "my_fs-cu132.sif.digest").write_text("sha256:1\n") + assert plan.image == str(tmp_path / "docker-my_fs-cu132.sif") + (tmp_path / "docker-my_fs-cu132.sif").write_bytes(b"") + (tmp_path / "docker-my_fs-cu132.sif.digest").write_text("sha256:1\n") own = {"kind": "docker", "value": "my_fs:cu132", "runtime": "apptainer"} assert decide(tmp_path, **own, local_id="sha256:1").build_from is None # a changed local Docker image is converted again assert decide(tmp_path, **own, local_id="sha256:2").build_from + # one whose name ends like a release is not taken for the installed release image, nor replaces it + (tmp_path / "fastsurfer-cu132-v2.6.0.sif").write_bytes(b"") + plan = decide(tmp_path, kind="docker", value="myrepo/fastsurfer:cu132-v2.6.0", runtime="apptainer") + assert plan.image == str(tmp_path / "docker-myrepo_fastsurfer-cu132-v2.6.0.sif") and plan.build_from def test_plan_paths_run(tmp_path): @@ -386,6 +404,59 @@ def test_plan_paths_brun(tmp_path): launcher.plan_paths(["--subjects_list", str(subjects)], tmp_path) +def test_plan_paths_long(tmp_path): + # long_fastsurfer.sh: several T1s (and T2s) after one flag, one per time point + data = tmp_path / "data" + data.mkdir() + for name in ("tp1.mgz", "tp2.mgz", "t2_1.mgz", "t2_2.mgz"): + (data / name).write_bytes(b"") + args = ["--tid", "person", "--t1s", "data/tp1.mgz", str(data / "tp2.mgz"), "--tpids", "tp1", "tp2", + "--t2s", "data/t2_1.mgz", "data/t2_2.mgz", "--sd", "out", "--log", "logs/long.log"] + out, mounts = launcher.plan_paths(args, tmp_path) + assert out == ["--tid", "person", "--t1s", str(data / "tp1.mgz"), str(data / "tp2.mgz"), "--tpids", "tp1", "tp2", + "--t2s", str(data / "t2_1.mgz"), str(data / "t2_2.mgz"), "--sd", str(tmp_path / "out"), + "--log", str(tmp_path / "logs/long.log")] + assert set(launcher.minimal_mounts(mounts)) == {data, tmp_path / "out", tmp_path / "logs"} + # every time point has to exist, not only the first + with pytest.raises(launcher.LauncherError, match="tp3.mgz"): + launcher.plan_paths(["--t1s", "data/tp1.mgz", "data/tp3.mgz", "--sd", "out"], tmp_path) + + +def test_exec_mounts(tmp_path): + work, data, results = tmp_path / "work", tmp_path / "data", tmp_path / "results" + for directory in (work, data, results): + directory.mkdir() + (data / "in.mgz").write_bytes(b"") + args = ["-v", str(data / "in.mgz"), f"--out={results / 'new.nii.gz'}", "relative.mgz", "2", + str(tmp_path / "missing" / "x.mgz")] + mounts = launcher.minimal_mounts(launcher.exec_mounts(args, work)) + # the current directory, the directory of an input, and that of an output in --flag=path form; a path + # whose directory does not exist is not taken for one, and relative paths stay in the current directory + assert mounts == sorted([work, data, results]) + + +def test_split_exec(): + argv, program_args = launcher.split_exec(["--runtime", "docker", "--exec", "mri_convert", "--help", "--image", "x"]) + options, rest = launcher.make_parser().parse_known_args(argv) + # the program's own flags, even the launcher's --help and --image, reach the program + assert (options.runtime, options.exec, rest) == ("docker", "mri_convert", []) + assert program_args == ["--help", "--image", "x"] + # the same with --exec=, also without arguments for it + argv, program_args = launcher.split_exec(["--exec=nib-convert", "in.mgz", "out.nii.gz"]) + assert (launcher.make_parser().parse_known_args(argv)[0].exec, program_args) == ("nib-convert", + ["in.mgz", "out.nii.gz"]) + assert launcher.split_exec(["--exec=bash"]) == (["--exec=bash"], []) + + +def test_fastsurfer_script(): + # the FastSurfer scripts by name or by their path in the image; anything else is another program + assert launcher.fastsurfer_script("brun_fastsurfer.sh") == launcher.BRUN_SCRIPT + assert launcher.fastsurfer_script("/fastsurfer/long_fastsurfer.sh") == "/fastsurfer/long_fastsurfer.sh" + assert launcher.fastsurfer_script("run_fastsurfer.sh") == launcher.RUN_SCRIPT + for other in ("nib-convert", "/somewhere/else/brun_fastsurfer.sh", "recon-surf.sh"): + assert launcher.fastsurfer_script(other) is None + + def test_link_directories(tmp_path): # a chain of links, as in a DataLad dataset: the link, a relative link in another directory, the file for name in ("data", "annex/key", "objects/abc"): @@ -436,9 +507,194 @@ def test_build_image(tmp_path): assert sorted(p.name for p in sif.parent.iterdir()) == sorted([sif.name, failed.name]) +def amd_topology(tmp_path, nodes): + """A stand-in for the kernel driver's /dev/kfd and topology: one node per (gfx_target_version, GiB).""" + device, topology = tmp_path / "kfd", tmp_path / "nodes" + device.write_text("") + for number, (target, gib) in enumerate(nodes): + node = topology / str(number) + (node / "mem_banks" / "0").mkdir(parents=True) + (node / "properties").write_text(f"cpu_cores_count 0\ngfx_target_version {target}\n") + (node / "mem_banks" / "0" / "properties").write_text(f"heap_type 1\nsize_in_bytes {gib * 2**30}\n") + return device, topology + + +def test_query_amd_gpus(tmp_path): + # node 0 is the CPU, without a GPU architecture; the GPUs are numbered from 0 as ROCm does + device, topology = amd_topology(tmp_path, [(0, 64), (100302, 8), (110000, 24)]) + gpus = launcher.query_amd_gpus(device, topology) + assert [(g.index, g.arch, g.memory_gib) for g in gpus] == [(0, "gfx1032", 8), (1, "gfx1100", 24)] + # without the kernel driver, no AMD GPU + assert launcher.query_amd_gpus(tmp_path / "missing", topology) == [] + + +@pytest.mark.parametrize( + ("version", "override"), + [ + ((11, 0, 0), ""), # gfx1100, in the image + ((9, 0, 10), ""), # gfx90a, the stepping is hexadecimal + ((10, 3, 2), "10.3.0"), # gfx1032 (Radeon Pro W6600), runs as gfx1030 + ((9, 0, 12), "9.0.0"), # gfx90c, an APU, runs as gfx900 + ((8, 0, 3), None), # gfx803, no image of its family + ], +) +def test_amd_override(version, override): + assert launcher.amd_override(launcher.AmdGpu(0, version)) == override + + +def test_plan_amd_build(): + w6600, rx7900 = launcher.AmdGpu(0, (10, 3, 2), 8), launcher.AmdGpu(1, (11, 0, 0), 24) + # the GPU with the most memory, shown alone to the container, without an override + build = launcher.plan_amd_build([], [w6600, rx7900], False) + assert (build.prefix, build.gpu, build.amd, build.devices, build.env) == ("rocm7.2", True, True, [1], []) + assert "experimental" in build.notes[0] + # a GPU of a supported family runs with the override, which says so + alone = launcher.plan_amd_build([], [w6600], False) + assert alone.env == ["HSA_OVERRIDE_GFX_VERSION=10.3.0"] and "gfx1030" in alone.notes[1] + # with --brun, only GPUs that run with the same override, as it applies to the whole container + assert launcher.plan_amd_build([], [w6600, rx7900], True).devices == [1] + # named GPUs as named; one that does not exist stops + assert launcher.plan_amd_build(["--device", "cuda:0"], [w6600, rx7900], False).devices is None + with pytest.raises(launcher.LauncherError, match="no AMD GPU 2"): + launcher.plan_amd_build(["--device", "cuda:2"], [w6600, rx7900], False) + # no supported GPU: the CPU, or an error if a GPU was requested + old = launcher.AmdGpu(0, (8, 0, 3), 4) + assert not launcher.plan_amd_build([], [old], False).gpu + with pytest.raises(launcher.LauncherError): + launcher.plan_amd_build(["--device", "cuda"], [old], False) + + +@pytest.mark.parametrize( + ("error", "system", "wsl", "advice"), + [ + # the errors Docker prints, and what the launcher tells the user to do instead + ("permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: " + "Get \"http://%2Fvar%2Frun%2Fdocker.sock/v1.45/info\": dial unix /var/run/docker.sock: connect: " + "permission denied", "Linux", False, "docker group"), + ("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?", + "Linux", False, "administrator starts it"), + ("Cannot connect to the Docker daemon at unix:///Users/me/.docker/run/docker.sock. Is the docker daemon " + "running?", "Darwin", False, "start Docker Desktop"), + ("The command 'docker' could not be found in this WSL 2 distro.\nWe recommend to activate the WSL " + "integration in Docker Desktop settings.", "Linux", True, "WSL integration"), + ("error during connect: something else\n", "Linux", False, "does not work: error during connect"), + ], +) +def test_docker_problem(error, system, wsl, advice): + assert advice in launcher.docker_problem(error, system, wsl) + + +def test_plan_image_build_custom(): + nvidia, amd = [gpu((8, 6))], [launcher.AmdGpu(0, (11, 0, 0), 24)] + # an own image on a machine with only AMD GPUs gets them, whatever its name + build = launcher.plan_image_build("docker", "my_fastsurfer:dev", [], [], amd, False) + assert (build.prefix, build.gpu, build.amd) == ("custom", True, True) + assert launcher.plan_image_build("sif", "/images/own.sif", [], [], amd, False).amd + # with an NVIDIA GPU, the NVIDIA one, unless the name says ROCm + build = launcher.plan_image_build("docker", "my_fastsurfer:dev", [], nvidia, amd, False) + assert (build.gpu, build.amd) == (True, False) + assert launcher.plan_image_build("docker", "my_fastsurfer:rocm-dev", [], nvidia, amd, False).amd + # --device cpu gives the container no GPU + assert not launcher.plan_image_build("docker", "my_fastsurfer:dev", ["--device", "cpu"], [], amd, False).gpu + + +def test_plan_image_build_rocm_on_cpu(): + # an installed ROCm image also runs on the CPU, without an AMD GPU, with a note unless the CPU was asked for + for kind, value in (("rocm", ""), ("tag", "rocm7.2-v2.6.0")): + build = launcher.plan_image_build(kind, value, [], [gpu((8, 6))], [], False) + assert (build.prefix, build.gpu, build.amd) == ("rocm7.2", False, False) and "CPU" in build.notes[0] + build = launcher.plan_image_build(kind, value, ["--device", "cpu"], [], [], False) + assert (build.prefix, build.gpu, build.notes) == ("rocm7.2", False, []) + # a GPU requested for it without an AMD GPU stops + with pytest.raises(launcher.LauncherError, match="No AMD GPU"): + launcher.plan_image_build("rocm", "", ["--device", "cuda"], [], [], False) + + +def test_decide_image_rocm(tmp_path): + # for an AMD GPU only the ROCm image counts among the installed ones, else the newest ROCm release + hub = {**HUB, "rocm7.2-v2.6.0": {"sha256:r"}} + installed = {"rocm7.2-v2.6.0": "sha256:r", "cu132-v2.6.0": "sha256:a"} + plan = launcher.decide_image("auto", "", "rocm7.2", "docker", BOTH, installed, {}, hub, tmp_path, False) + assert plan.image == "deepmi/fastsurfer:rocm7.2-v2.6.0" and not plan.pull + plan = launcher.decide_image("auto", "", "rocm7.2", "docker", BOTH, {"cu132-v2.6.0": "sha256:a"}, {}, hub, + tmp_path, False) + assert plan.image == "deepmi/fastsurfer:rocm7.2-v2.6.0" + + +def test_parse_image_spec_rocm(): + assert launcher.parse_image_spec("rocm") == ("rocm", "") + assert launcher.parse_image_spec("rocm7.2-v2.6.0") == ("tag", "rocm7.2-v2.6.0") + + +def test_amd_commands(tmp_path): + env = ["HSA_OVERRIDE_GFX_VERSION=10.3.0"] + docker = launcher.docker_command("img:tag", True, [], launcher.RUN_SCRIPT, [], [], "1000:1000", devices=[1], + amd=True, env=env, groups=[44, 109]) + assert "--gpus" not in docker + start = docker.index("--device") + assert docker[start:start + 15] == ["--device", "/dev/kfd", "--device", "/dev/dri", "--ipc=host", "--shm-size", + "8G", "--group-add", "44", "--group-add", "109", "--env", env[0], "--env", + "HIP_VISIBLE_DEVICES=1"] + apptainer = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], launcher.RUN_SCRIPT, [], [], + devices=[1], amd=True, env=env) + assert apptainer[2:7] == ["--rocm", "--env", env[0], "--env", "HIP_VISIBLE_DEVICES=1"] + + +def macos_applications(tmp_path, names): + """A stand-in for /Applications with FastSurfer packages, each a folder with the setup script.""" + for name in names: + (tmp_path / name).mkdir() + (tmp_path / name / launcher.MACOS_SETUP).write_text("") + return tmp_path + + +def test_macos_packages(tmp_path): + # releases by version; the app that opens the console and a development build do not count + apps = macos_applications(tmp_path, ["FastSurfer2.5.4", "FastSurfer2.6.0", "FastSurfer2.7.0-dev0"]) + (tmp_path / "FastSurfer2.6.0.app").mkdir() + assert launcher.macos_packages(apps) == {"2.5.4": apps / "FastSurfer2.5.4", "2.6.0": apps / "FastSurfer2.6.0"} + # --image with the folder of a package names it directly + dev = apps / "FastSurfer2.7.0-dev0" + assert launcher.parse_image_spec(str(dev)) == ("macos", str(dev.resolve())) + + +def test_decide_macos_package(tmp_path): + packages = {"2.5.4": tmp_path / "FastSurfer2.5.4", "2.6.0": tmp_path / "FastSurfer2.6.0"} + # the same rules as for images: several installed stop the launcher, newest first + with pytest.raises(launcher.LauncherError, match="2.6.0, 2.5.4.*--image 2.6.0"): + launcher.decide_macos_package("auto", "", packages, "2.6.0") + home, notes = launcher.decide_macos_package("version", "2.5.4", packages, "2.6.0") + assert home == packages["2.5.4"] and "--image 2.5.4" in notes[0][1] + # one installed runs, with a note on a newer release + home, notes = launcher.decide_macos_package("auto", "", {"2.5.4": packages["2.5.4"]}, "2.6.0") + assert home == packages["2.5.4"] and any("2.6.0 is available" in note for _, note in notes) + # another program takes the newest, without study advice + home, notes = launcher.decide_macos_package("auto", "", packages, "2.6.0", other_program=True) + assert home == packages["2.6.0"] and not any("For a study" in note for _, note in notes) + # none installed, a version not installed, and a container image: what to do instead + with pytest.raises(launcher.LauncherError, match="--update"): + launcher.decide_macos_package("auto", "", {}, "2.6.0") + with pytest.raises(launcher.LauncherError, match="--update"): + launcher.decide_macos_package("version", "2.4.2", packages, "2.6.0") + with pytest.raises(launcher.LauncherError, match="--runtime docker"): + launcher.decide_macos_package("tag", "cu132-v2.6.0", packages, "2.6.0") + + +def test_macos_command(tmp_path): + # the package's setup script, then the program with its arguments unchanged, as bash runs it + command = launcher.macos_command(tmp_path, "nib-convert", ["a b.mgz", "out.nii.gz"]) + setup = tmp_path / launcher.MACOS_SETUP + setup.write_text("export STAND_IN=1\n") + probe = launcher.macos_command(tmp_path, sys.executable, ["-c", "import os,sys; print(os.environ['STAND_IN'], " + "sys.argv[1:])", "a b.mgz"]) + assert command[-3:] == ["nib-convert", "a b.mgz", "out.nii.gz"] + assert subprocess.run(probe, capture_output=True, text=True).stdout == "1 ['a b.mgz']\n" + + def test_commands(tmp_path): args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] - docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") + run = launcher.RUN_SCRIPT + docker = launcher.docker_command("img:tag", True, [tmp_path], run, args, [], "1000:1000") # --init, so Ctrl-C and a cancelled job reach FastSurfer init = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1"] # the host's time zone, on Linux @@ -447,18 +703,31 @@ def test_commands(tmp_path): assert docker == init + ["--user", "1000:1000", "--gpus", "all", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID"] + zone + [ "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args # selected GPUs, quoted so Docker does not split the list - selected = launcher.docker_command("img:tag", True, [], "run_fastsurfer.sh", [], [], "1:1", devices=[1, 3]) + selected = launcher.docker_command("img:tag", True, [], run, [], [], "1:1", devices=[1, 3]) assert selected[8:10] == ["--gpus", '"device=1,3"'] - sif_gpu = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], "run_fastsurfer.sh", [], [], - devices=[2]) + sif_gpu = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], run, [], [], devices=[2]) assert sif_gpu[2:8] == ["--nv", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", "--env", "CUDA_VISIBLE_DEVICES=2", "--no-mount"] - emulated = launcher.docker_command("img:tag", False, [tmp_path], "run_fastsurfer.sh", args, [], "501:20", True) + emulated = launcher.docker_command("img:tag", False, [tmp_path], run, args, [], "501:20", True) assert emulated[:10] == init + ["--user", "501:20", "--platform", "linux/amd64"] - brun = launcher.docker_command("img:tag", False, [tmp_path], "brun_fastsurfer.sh", args, [], "1000:1000") - assert brun[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", "img:tag", - "/fastsurfer/brun_fastsurfer.sh"] + # the other scripts and programs run through the image's entrypoint, which sets up the environment + for program in (launcher.BRUN_SCRIPT, launcher.SCRIPTS["long_fastsurfer.sh"], "nib-convert"): + other = launcher.docker_command("img:tag", False, [tmp_path], program, args, [], "1000:1000") + assert other[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", + "img:tag", program] sif = tmp_path / "fs.sif" - apptainer = launcher.apptainer_command("apptainer", sif, False, [tmp_path], "run_fastsurfer.sh", args, []) + apptainer = launcher.apptainer_command("apptainer", sif, False, [tmp_path], run, args, []) assert apptainer == ["apptainer", "exec", "--no-mount", "home,cwd", "-e", "-B", str(tmp_path), str(sif), "/fastsurfer/run_fastsurfer.sh"] + args + # --exec: in the current directory, and with Docker attached to the terminal if there is one + exec_args = ["in.mgz", "out.nii.gz"] + in_dir = launcher.apptainer_command("apptainer", sif, False, [tmp_path], "nib-convert", exec_args, [], + workdir=tmp_path) + assert in_dir[-6:] == ["--pwd", str(tmp_path), str(sif), "nib-convert"] + exec_args + docker_exec = launcher.docker_command("img:tag", False, [tmp_path], "bash", [], [], "1:1", workdir=tmp_path, + tty=True) + assert docker_exec[8] == "-it" + assert docker_exec[docker_exec.index("-w") + 1] == str(tmp_path) + # input from a pipe or file is passed on without a terminal, which Docker refuses for such input + piped = launcher.docker_command("img:tag", False, [tmp_path], "bash", [], [], "1:1", interactive=True) + assert piped[8] == "-i" diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer old mode 100644 new mode 100755 similarity index 54% rename from tools/fastsurfer_container.py rename to tools/fastsurfer index 746f428cd..107796a0b --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer @@ -14,22 +14,29 @@ # limitations under the License. """ -Run FastSurfer in its container image, without writing the container command. +Run FastSurfer with one command on every system: in its container image, or on a Mac with Apple silicon +with the macOS package. -Takes the flags of run_fastsurfer.sh (or, with --brun, of brun_fastsurfer.sh) and runs them in the -official image with Apptainer, Singularity or Docker, whichever is installed: +Takes the flags of run_fastsurfer.sh and runs them in the official image with Apptainer, Singularity +or Docker, whichever is installed: - python3 fastsurfer_container.py --t1 data/t1.nii.gz --sd results --sid subject1 + ./fastsurfer --t1 data/t1.nii.gz --sd results --sid subject1 + +--exec runs another program of the image with the arguments after it: brun_fastsurfer.sh and +long_fastsurfer.sh as FastSurfer itself, and any other program, e.g. --exec nib-convert in.mgz +out.nii.gz, in the current directory. It runs the FastSurfer release image installed on the machine (the newest release if there is none, -or the image --image names), picks the image type that fits the NVIDIA GPU and driver (or the CPU -image), downloads it once, +or the image --image names), picks the image type that fits the NVIDIA GPU and driver (or, without +one, the experimental ROCm image for an AMD GPU, or the CPU image), downloads it once, shares every file and directory the flags name with the container at the same path, passes the FreeSurfer license from $FS_LICENSE if --fs_license is not given, and runs the container as the -calling user. +calling user. On a Mac with Apple silicon, it runs the installed macOS package instead, with the +same choice among installed versions. Only the Python standard library is used, so this runs with the system Python of a cluster, -without FastSurfer installed. +without FastSurfer installed. It has no .py suffix, so a copy next to other Python code cannot +shadow the package fastsurfer. """ from __future__ import annotations @@ -47,11 +54,18 @@ DOCKER_REPO = "deepmi/fastsurfer" DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/CONTAINERS.html" -LAUNCHER_URL = "https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer_container.py" +# what to install on each system, for the messages about a missing or unusable container runtime +SETUP_URL = "https://deep-mi.org/FastSurfer/stable/overview/QUICKSTART.html#set-up-your-computer" +LAUNCHER_URL = "https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer" # seconds to wait for Docker Hub and GitHub, so a machine without internet is not held up HUB_TIMEOUT = 5 MACOS_PACKAGE_URL = "https://github.com/Deep-MI/FastSurfer/releases/latest/download/FastSurfer-macos-darwin_arm64.pkg" MACOS_DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/install/MACOS.html" +# the macOS package installs into /Applications/FastSurfer, with a script that sets up its +# Python, FreeSurfer and PATH, as the console it opens does +MACOS_APPLICATIONS = Path("/Applications") +MACOS_SETUP = "macos_setup_fastsurfer.sh" +GITHUB_RELEASES = "https://api.github.com/repos/Deep-MI/FastSurfer/releases" # the 15 GB the macOS page asks Docker to have, less what the kernel of Docker's VM keeps for itself MIN_DOCKER_MEMORY_GIB = 14 @@ -66,15 +80,30 @@ # the graphics memory FastSurfer needs at 0.8 mm, see the system requirements in the README MIN_GPU_MEMORY_GIB = 8 +# The ROCm images for AMD GPUs (experimental): (ROCm version as in tool.rocm of pyproject.toml, tag +# prefix), and the GPU architectures their PyTorch is built for, from torch.cuda.get_arch_list() in it +ROCM_BUILD = ((7, 2), "rocm7.2") +ROCM_ARCHS = {"gfx900", "gfx906", "gfx908", "gfx90a", "gfx942", "gfx950", "gfx1030", "gfx1100", "gfx1101", + "gfx1102", "gfx1103", "gfx1150", "gfx1151", "gfx1200", "gfx1201"} +# where AMD's kernel driver lists the GPUs, which the containers need, not ROCm on the host +KFD_DEVICE = Path("/dev/kfd") +KFD_TOPOLOGY = Path("/sys/class/kfd/kfd/topology/nodes") + # flags of run_fastsurfer.sh and brun_fastsurfer.sh whose value is a file that has to exist INPUT_FLAGS = {"--t1", "--t2", "--lesion_mask", "--fs_license", "--subjects_list", "--subject_list"} +# flags of long_fastsurfer.sh followed by several files that have to exist, one per time point +MULTI_INPUT_FLAGS = {"--t1s", "--t2s"} # flags whose value is a file or directory FastSurfer writes, so its directory is created if missing OUTPUT_FLAGS = { "--sd", "--seg_log", "--conformed_name", "--norm_name", "--norm_name_t2", "--seg", "--asegdkt_segfile", "--aparc_aseg_segfile", "--asegdkt_statsfile", "--aseg_statsfile", "--aseg_segfile", "--mask_name", "--cereb_segfile", "--cereb_statsfile", "--hypo_segfile", - "--hypo_statsfile", "--statusfile", + "--hypo_statsfile", "--statusfile", "--log", } +# the FastSurfer scripts in the image: run_fastsurfer.sh by default, the others with --exec +RUN_SCRIPT = "/fastsurfer/run_fastsurfer.sh" +BRUN_SCRIPT = "/fastsurfer/brun_fastsurfer.sh" +SCRIPTS = {script.rsplit("/", 1)[-1]: script for script in (RUN_SCRIPT, BRUN_SCRIPT, "/fastsurfer/long_fastsurfer.sh")} class LauncherError(Exception): @@ -91,14 +120,35 @@ class Gpu: memory_gib: float | None = None +@dataclass +class AmdGpu: + """An AMD GPU as the kernel driver lists it, numbered as ROCm numbers the GPUs.""" + + index: int + # the architecture as (major, minor, stepping), e.g. (10, 3, 2) for gfx1032 + version: tuple[int, int, int] + memory_gib: float | None = None + + @property + def arch(self) -> str: + major, minor, stepping = self.version + return f"gfx{major}{minor:x}{stepping:x}" + + @dataclass class Build: - """The image build to run, whether the container gets GPUs, and which (None for all).""" + """ + The image build to run, whether the container gets GPUs, and which (None for all). + + `amd` marks the GPUs as AMD GPUs, and `env` holds the variables the container needs for them. + """ prefix: str gpu: bool notes: list[str] = field(default_factory=list) devices: list[int] | None = None + amd: bool = False + env: list[str] = field(default_factory=list) def parse_version(version: str) -> tuple[int, ...] | None: @@ -356,7 +406,7 @@ def gpu_request(args: list[str], brun: bool = False) -> tuple[str, list[int] | N several = [v for v in cuda if v.startswith("cuda:") and not re.fullmatch(r"cuda:\d+", v)] if several and not brun: raise LauncherError(f"{several[0]}: run_fastsurfer.sh uses one GPU, e.g. cuda:1. Lists and ranges of " - f"GPUs are for processing several subjects with --brun.") + f"GPUs are for processing several subjects with --exec brun_fastsurfer.sh.") named = [i for v in cuda for i in (cuda_indices(v) or [])] if named and values[0] in ("auto", "cuda"): # only the view aggregation names a GPU; the segmentation then runs on the first one @@ -419,6 +469,142 @@ def plan_build_for_type(args: list[str], gpus: list[Gpu], brun: bool, image_type devices=[gpu.index for gpu in chosen]) +def _properties(path: Path) -> dict[str, str]: + """A properties file of the kernel driver's topology, lines of " ".""" + return dict(line.split()[:2] for line in path.read_text().splitlines() if len(line.split()) >= 2) + + +def query_amd_gpus(device: Path = KFD_DEVICE, topology: Path = KFD_TOPOLOGY) -> list[AmdGpu]: + """ + The AMD GPUs, from the kernel driver's topology, in the order ROCm numbers them; empty without the + driver. The topology also lists the CPUs, as nodes without a GPU architecture. + """ + if not device.exists() or not topology.is_dir(): + return [] + gpus = [] + for node in sorted((n for n in topology.iterdir() if n.name.isdigit()), key=lambda n: int(n.name)): + try: + target = int(_properties(node / "properties").get("gfx_target_version", "0")) + banks = sorted((node / "mem_banks").iterdir()) if (node / "mem_banks").is_dir() else [] + sizes = [int(_properties(bank / "properties").get("size_in_bytes", "0")) for bank in banks] + except (OSError, ValueError): + continue + if target: + # e.g. 100302 for gfx1032 + version = (target // 10000, target // 100 % 100, target % 100) + gpus.append(AmdGpu(len(gpus), version, sum(sizes) / 2**30 if sizes else None)) + return gpus + + +def amd_override(gpu: AmdGpu) -> str | None: + """ + The HSA_OVERRIDE_GFX_VERSION that makes the ROCm image run on this GPU: "" if it needs none, the version + of the first architecture of its family if the image supports that one (e.g. 10.3.0 for gfx1032), and + None if the image does not support the GPU. + """ + if gpu.arch in ROCM_ARCHS: + return "" + major, minor, _ = gpu.version + if AmdGpu(gpu.index, (major, minor, 0)).arch in ROCM_ARCHS: + return f"{major}.{minor}.0" + return None + + +def describe_amd(gpu: AmdGpu) -> str: + """E.g. "GPU 0 (gfx1100, 24 GB)".""" + memory = f", {gpu.memory_gib:.0f} GB" if gpu.memory_gib is not None else "" + return f"GPU {gpu.index} ({gpu.arch}{memory})" + + +def plan_amd_build(args: list[str], gpus: list[AmdGpu], brun: bool, image_type: str = ROCM_BUILD[1]) -> Build: + """ + The ROCm image and the AMD GPUs for these arguments (experimental). + + As for NVIDIA GPUs, named GPUs are used as named, and otherwise the one with the most memory; with + brun_fastsurfer.sh, every GPU that runs with the same HSA_OVERRIDE_GFX_VERSION, as it applies to all. + """ + use, named = gpu_request(args, brun) + if use == "cpu": + return Build(image_type, False) + if named is not None: + by_index = {gpu.index: gpu for gpu in gpus} + missing = [i for i in named if i not in by_index] + if missing: + raise LauncherError(f"There is no AMD GPU {missing[0]}, the driver lists {len(gpus)}.") + chosen = [by_index[i] for i in named] + unsupported = [gpu for gpu in chosen if amd_override(gpu) is None] + if unsupported: + raise LauncherError(f"The ROCm image does not support the AMD {describe_amd(unsupported[0])}.") + if len({amd_override(gpu) for gpu in chosen}) > 1: + raise LauncherError("The named AMD GPUs are of different kinds, which cannot run together; name GPUs " + "of one kind.") + devices = None + else: + supported = [gpu for gpu in gpus if amd_override(gpu) is not None] + if not supported: + reason = ("No AMD GPU found" if not gpus else + f"The ROCm image supports none of the AMD GPUs here ({', '.join(map(describe_amd, gpus))})") + if use == "cuda": + raise LauncherError(f"{reason}.") + return Build(image_type, False, [f"{reason}, so FastSurfer runs on the CPU."]) + best = max(supported, key=lambda gpu: ((gpu.memory_gib or 0) >= MIN_GPU_MEMORY_GIB, gpu.memory_gib or 0)) + chosen = [gpu for gpu in supported if amd_override(gpu) == amd_override(best)] if brun else [best] + devices = [gpu.index for gpu in chosen] + override = amd_override(chosen[0]) + notes = [f"Using the AMD {', '.join(map(describe_amd, chosen))} with the ROCm image, which is experimental; " + f"--device cpu runs on the CPU instead."] + env = [] + if override: + env.append(f"HSA_OVERRIDE_GFX_VERSION={override}") + family = AmdGpu(0, chosen[0].version[:2] + (0,)).arch + notes.append(f"The ROCm image is not built for {chosen[0].arch}, so it runs the GPU as {family} " + f"(HSA_OVERRIDE_GFX_VERSION={override}), which works for many GPUs of this family.") + return Build(image_type, True, notes, devices, amd=True, env=env) + + +def plan_image_build(kind: str, value: str, request: list[str], gpus: list[Gpu], amd_gpus: list[AmdGpu], + brun: bool) -> Build: + """ + The image type and the GPUs for what --image names (see parse_image_spec) and these arguments. + + AMD GPUs (experimental) are used for a ROCm image, named with --image, and when there is no NVIDIA GPU. + A ROCm image without a usable AMD GPU runs on the CPU, unless a GPU is requested. An own image (a Docker + image or .sif file) gets the GPUs of this machine: the AMD ones if its name says ROCm or there is no + NVIDIA GPU, else the NVIDIA ones. + """ + custom = kind in ("sif", "docker") + image_type = RELEASE_TAG.fullmatch(value).group(1) if kind in ("tag", "release") else None + wants_gpu = gpu_request(request, brun)[0] != "cpu" + amd_runs = any(amd_override(gpu) is not None for gpu in amd_gpus) + rocm = kind == "rocm" or (image_type or "").startswith("rocm") or (custom and "rocm" in value.lower()) + if rocm or (custom and amd_gpus and not gpus and wants_gpu): + build = plan_amd_build(request, amd_gpus, brun, image_type or ROCM_BUILD[1]) + if custom: + build.prefix = "custom" + return build + if custom: + return Build("custom", wants_gpu and bool(gpus)) + if kind in ("tag", "release"): + return plan_build_for_type(request, gpus, brun, image_type) + if not gpus and amd_runs and wants_gpu: + return plan_amd_build(request, amd_gpus, brun) + build = plan_build(request, gpus, brun) + if gpus and amd_runs: + build.notes.append("This machine also has an AMD GPU; --image rocm runs FastSurfer on it instead " + "(experimental).") + elif amd_gpus and not amd_runs: + build.notes.append(f"The ROCm image supports none of the AMD GPUs here " + f"({', '.join(map(describe_amd, amd_gpus))}).") + return build + + +def amd_device_groups(device: Path = KFD_DEVICE, dri: Path = Path("/dev/dri")) -> list[int]: + """The groups that own the devices of AMD GPUs, which a Docker container running as the user needs.""" + # the render nodes of the GPUs + paths = [device] + (sorted(dri.glob("render*")) if dri.is_dir() else []) + return sorted({path.stat().st_gid for path in paths if path.exists()}) + + def image_reference(prefix: str, version: str) -> str: """The Docker image to run, e.g. deepmi/fastsurfer:cu132-v2.6.0.""" return f"{DOCKER_REPO}:{prefix}-v{version}" @@ -429,6 +615,26 @@ def image_reference(prefix: str, version: str) -> str: SIF_PREFIX = DOCKER_REPO.rsplit("/", 1)[-1] + "-" +# the root certificates macOS ships; Python from python.org has none of its own until its "Install +# Certificates" script runs, and fails every HTTPS request until then +SYSTEM_CA_FILE = "/etc/ssl/cert.pem" + + +def open_url(url: str): + """urllib's urlopen, retried with the system's certificates if Python cannot verify the server.""" + import ssl + import urllib.error + import urllib.request + + try: + return urllib.request.urlopen(url, timeout=HUB_TIMEOUT) + except urllib.error.URLError as error: + if not isinstance(error.reason, ssl.SSLCertVerificationError) or not os.path.isfile(SYSTEM_CA_FILE): + raise + context = ssl.create_default_context(cafile=SYSTEM_CA_FILE) + return urllib.request.urlopen(url, timeout=HUB_TIMEOUT, context=context) + + def hub_tags() -> dict[str, set[str]] | None: """ The release tags on Docker Hub and the digests each is known by, None if Docker Hub cannot be reached. @@ -438,13 +644,12 @@ def hub_tags() -> dict[str, set[str]] | None: recorded. """ import json - import urllib.request url = f"https://hub.docker.com/v2/repositories/{DOCKER_REPO}/tags?page_size=100&name=-v" tags: dict[str, set[str]] = {} try: while url and len(tags) < 2000: - with urllib.request.urlopen(url, timeout=HUB_TIMEOUT) as response: + with open_url(url) as response: page = json.load(response) for result in page.get("results", []): digests = {result.get("digest")} | {image.get("digest") for image in result.get("images", [])} @@ -496,6 +701,14 @@ def sif_path(image_dir: Path, tag: str) -> Path: return image_dir / f"{SIF_PREFIX}{tag}.sif" +def converted_sif(image_dir: Path, reference: str) -> Path: + """ + Where the Apptainer image converted from another Docker image is kept, named after its full reference, + e.g. docker-myrepo_fastsurfer-cu132.sif, so it cannot take the place of an official release image. + """ + return image_dir / f"docker-{re.sub(r'[/:@]', lambda m: '_' if m.group() == '/' else '-', reference)}.sif" + + def _recorded(sif: Path) -> str: """What the launcher wrote next to an image it built: the digest or image ID of its source, or "".""" record = sif.with_name(sif.name + ".digest") @@ -516,12 +729,17 @@ def parse_image_spec(spec: str | None) -> tuple[str, str]: Returns ------- tuple[str, str] - ("auto", "") without --image, ("version", "2.6.0"), ("tag", "cu132-v2.6.0") for a release tag, - ("release", "cu132-v2.6.0") for the full reference deepmi/fastsurfer:cu132-v2.6.0, ("sif", path) - for an Apptainer file and ("docker", reference) for any other Docker image. + ("auto", "") without --image, ("rocm", "") for the newest ROCm image, ("version", "2.6.0"), + ("tag", "cu132-v2.6.0") for a release tag, ("release", "cu132-v2.6.0") for the full reference + deepmi/fastsurfer:cu132-v2.6.0, ("sif", path) for an Apptainer file, ("macos", folder) for a macOS + package and ("docker", reference) for any other Docker image. """ if not spec: return "auto", "" + if spec == "rocm": + return "rocm", "" + if (Path(spec).expanduser() / MACOS_SETUP).is_file(): + return "macos", str(Path(spec).expanduser().resolve()) if spec.endswith(".sif"): return "sif", spec if parse_version(spec): @@ -562,7 +780,8 @@ def _apptainer_runtime(runtime: str, installed: set[str]) -> str | None: def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: set[str], docker_images: dict[str, str] | None, sif_images: dict[str, str], hub: dict[str, set[str]] | None, image_dir: Path, update: bool, - local_id: str | None = None, gpus: list[Gpu] | None = None) -> ImagePlan: + local_id: str | None = None, gpus: list[Gpu] | None = None, + other_program: bool = False) -> ImagePlan: """ Which image to run, with which runtime, and whether it has to be downloaded or built first. @@ -597,6 +816,9 @@ def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: se gpus : list[Gpu], None The GPUs the run will use, to tell which installed images run here; empty for a run on the CPU. None counts only images of type `prefix`. + other_program : bool, default=False + For a program other than FastSurfer (--exec), whose results do not depend on the image: use the + newest of several installed images instead of stopping, and give no advice on keeping the image. """ apptainer = _apptainer_runtime(runtime, installed) docker_images = docker_images or {} @@ -654,7 +876,7 @@ def release_exists(tag: str) -> None: if runtime == "docker" or (runtime == "auto" and (local_id is not None or apptainer is None)): return ImagePlan(require("docker"), value, pull=update) # Apptainer runs a Docker image as a .sif file, converted from the local Docker if it is there - sif = image_dir / (value.rsplit("/", 1)[-1].replace(":", "-") + ".sif") + sif = converted_sif(image_dir, value) source = f"docker-daemon://{value}" if local_id else f"docker://{value}" stale = sif.is_file() and local_id is not None and _recorded(sif) != local_id build = update or not sif.is_file() or stale @@ -675,6 +897,10 @@ def runs_here(image_type: str) -> bool: candidates = sorted({t for t in [*docker_images, *sif_images] if (m := RELEASE_TAG.fullmatch(t)) and runs_here(m.group(1))}, key=lambda t: (parse_version(t.rsplit("-v", 1)[-1]), t)) + # a runtime named with --runtime counts its own images, and the others only if it has none + own_store = {"docker": docker_images, "apptainer": sif_images, "singularity": sif_images}.get(runtime) + if own_store is not None and any(t in own_store for t in candidates): + candidates = [t for t in candidates if t in own_store] newest = newest_version(list(hub), prefix) if hub is not None else None if update or not candidates: if newest is None: @@ -697,6 +923,11 @@ def runs_here(image_type: str) -> bool: if newest is not None and parse_version(newest) > parse_version(version): plan.notes.append(("INFO", f"FastSurfer {newest} is available ({prefix}-v{newest}); --update " f"switches to it.")) + elif other_program: + tag = candidates[-1] + plan = place_release(tag, checked=True) + plan.notes.append(("INFO", f"Several FastSurfer images are installed, this uses the newest, {tag}; " + f"choose another with --image .")) else: def stores(t: str) -> str: return " and ".join(s for s, has in (("Docker", t in docker_images), ("Apptainer", t in sif_images)) if has) @@ -711,8 +942,9 @@ def stores(t: str) -> str: hint = f"e.g. --image {candidates[-1]}" raise LauncherError(f"Several FastSurfer images that run here are installed: {'; '.join(where)}. Choose " f"one with --image , {hint}.") - plan.notes.insert(0, ("INFO", f"Running FastSurfer {plan.tag.rsplit('-v', 1)[-1]} ({plan.tag}). For a study, " - f"keep this image with --image {plan.tag}.")) + if not other_program: + plan.notes.insert(0, ("INFO", f"Running FastSurfer {plan.tag.rsplit('-v', 1)[-1]} ({plan.tag}). For a " + f"study, keep this image with --image {plan.tag}.")) return plan @@ -756,7 +988,7 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st Parameters ---------- args : list[str] - The arguments for run_fastsurfer.sh or brun_fastsurfer.sh. + The arguments for run_fastsurfer.sh, brun_fastsurfer.sh or long_fastsurfer.sh. cwd : Path The directory relative paths are relative to. create : bool, default=True @@ -785,6 +1017,15 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st mounts.extend(_subjects_list_mounts(path, create)) out.append(str(path)) i += 1 + elif flag in MULTI_INPUT_FLAGS: + # long_fastsurfer.sh: one path per time point, until the next flag + while i < len(args) and not args[i].startswith("-"): + path = _absolute(args[i], cwd) + if not path.exists(): + raise LauncherError(f"{flag} {args[i]}: no such file.") + mounts.extend(_link_directories(path)) + out.append(str(path)) + i += 1 elif flag == "--subjects": # brun_fastsurfer.sh: = ... until the next flag while i < len(args) and not args[i].startswith("--"): @@ -800,6 +1041,27 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st return out, mounts +def exec_mounts(args: list[str], cwd: Path) -> list[Path]: + """ + The directories to share for any program: the current directory, where it runs, and those of its + absolute paths, also in `--flag=/path` form. + + An absolute path that exists is shared like an input, one whose directory exists like an output; + any other argument is not taken for a path. Relative paths only work inside the current directory. + """ + mounts = [cwd] + for arg in args: + value = arg.partition("=")[2] if arg.startswith("-") else arg + if not os.path.isabs(value): + continue + path = Path(os.path.normpath(value)) + if path.exists(): + mounts.extend(_link_directories(path)) + elif path.parent.is_dir(): + mounts.append(path.parent) + return mounts + + def _subjects_list_mounts(subjects_list: Path, create: bool = True) -> list[Path]: """ The directories to share for a brun_fastsurfer.sh subjects list, whose paths must be absolute. @@ -863,35 +1125,60 @@ def runtime_too_old(version_text: str) -> str | None: CUDA_ORDER = "CUDA_DEVICE_ORDER=PCI_BUS_ID" -def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], script: str, - args: list[str], extra: list[str], devices: list[int] | None = None) -> list[str]: - """The Apptainer/Singularity command, showing only `devices` if given; the home directory is not mounted.""" +def _gpu_env(amd: bool, devices: list[int] | None, env: list[str] | None) -> list[str]: + """The environment variables that select the GPUs in the container, for NVIDIA or AMD GPUs.""" + variables = list(env or []) if amd else [CUDA_ORDER] + if devices is not None: + variables.append(f"{'HIP' if amd else 'CUDA'}_VISIBLE_DEVICES={','.join(map(str, devices))}") + return [arg for variable in variables for arg in ("--env", variable)] + + +def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], program: str, + args: list[str], extra: list[str], devices: list[int] | None = None, + workdir: Path | None = None, amd: bool = False, env: list[str] | None = None) -> list[str]: + """ + The Apptainer/Singularity command running `program`, showing only `devices` if given and starting in + `workdir` if given; the home directory is not mounted. `amd` gives it AMD GPUs instead of NVIDIA ones, + with the variables in `env`. + """ cmd = [runtime, "exec"] if gpu: - cmd += ["--nv", "--env", CUDA_ORDER] - if devices is not None: - cmd += ["--env", f"CUDA_VISIBLE_DEVICES={','.join(map(str, devices))}"] + cmd += ["--rocm" if amd else "--nv"] + _gpu_env(amd, devices, env) cmd += ["--no-mount", "home,cwd", "-e"] for mount in mounts: cmd += ["-B", str(mount)] - return cmd + extra + [str(sif), f"/fastsurfer/{script}"] + args + if workdir is not None: + cmd += ["--pwd", str(workdir)] + return cmd + extra + [str(sif), program] + args -def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: list[str], +def docker_command(image: str, gpu: bool, mounts: list[Path], program: str, args: list[str], extra: list[str], user: str, emulate: bool = False, devices: list[int] | None = None, - ) -> list[str]: + workdir: Path | None = None, interactive: bool = False, amd: bool = False, + env: list[str] | None = None, groups: list[int] | None = None, tty: bool = False) -> list[str]: """ - The Docker command, running as the calling user and showing only `devices` if given. + The Docker command running `program`, as the calling user, showing only `devices` if given and + starting in `workdir` if given. - `emulate` runs the Intel image on Apple silicon. + `emulate` runs the Intel image on Apple silicon; `interactive` passes the input on (-i), and `tty` + gives the program a terminal (-t), e.g. for a shell. `amd` gives it AMD GPUs instead of NVIDIA ones, + with the variables in `env` and the user in `groups`, the groups that own the GPU devices. """ # --init: the image's entrypoint is a shell, which as the first process ignores Ctrl-C and the # signal a cancelled cluster job sends; tini passes them to all of FastSurfer's processes instead cmd = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1", "--user", user] - if gpu: + if interactive or tty: + cmd += ["-i" + ("t" if tty else "")] + if gpu and amd: + # the devices of the kernel driver, and the shared memory settings of AMD's documented PyTorch command + cmd += ["--device", "/dev/kfd", "--device", "/dev/dri", "--ipc=host", "--shm-size", "8G"] + for group in groups or []: + cmd += ["--group-add", str(group)] + cmd += _gpu_env(True, devices, env) + elif gpu: # the quotes are part of the value: without them, Docker splits a list of devices at the comma selection = "all" if devices is None else f'"device={",".join(map(str, devices))}"' - cmd += ["--gpus", selection, "--env", CUDA_ORDER] + cmd += ["--gpus", selection] + _gpu_env(False, None, None) if emulate: cmd += ["--platform", "linux/amd64"] if platform.system() == "Linux" and Path("/etc/localtime").exists(): @@ -899,13 +1186,100 @@ def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: cmd += ["-v", "/etc/localtime:/etc/localtime:ro"] for mount in mounts: cmd += ["-v", f"{mount}:{mount}"] + if workdir is not None: + cmd += ["-w", str(workdir)] cmd += extra - if script != "run_fastsurfer.sh": - # the image's entrypoint runs run_fastsurfer.sh; keep its environment setup for the other scripts - return cmd + ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", image, f"/fastsurfer/{script}"] + args + if program != RUN_SCRIPT: + # the image's entrypoint runs run_fastsurfer.sh; keep its environment setup for other programs + return cmd + ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", image, program] + args return cmd + [image] + args +def macos_packages(applications: Path = MACOS_APPLICATIONS) -> dict[str, Path]: + """The installed macOS packages of FastSurfer releases, by version: {"2.6.0": /Applications/FastSurfer2.6.0}.""" + packages = {} + for setup in applications.glob(f"FastSurfer*/{MACOS_SETUP}"): + version = setup.parent.name[len("FastSurfer"):] + if parse_version(version) is not None: + packages[version] = setup.parent + return packages + + +def decide_macos_package(kind: str, value: str, packages: dict[str, Path], newest: str | None, + other_program: bool = False) -> tuple[Path, list[tuple[str, str]]]: + """ + The macOS package to run, and notes for the user, with the rules for container images: one installed + release runs, several stop the launcher (a program other than FastSurfer takes the newest), --image + chooses one, and --image names a package directly. + + Parameters + ---------- + kind, value : str + What --image names, see parse_image_spec. + packages : dict[str, Path] + The installed packages, see macos_packages. + newest : str, None + The newest release, None if Docker Hub cannot be reached. + other_program : bool, default=False + For a program other than FastSurfer (--exec), whose results do not depend on the version. + """ + install = "--update downloads it and opens the macOS Installer" + if kind == "macos": + return Path(value), [] + if kind not in ("auto", "version"): + raise LauncherError(f"--image {value or kind} names a container image, but on a Mac with Apple silicon " + f"FastSurfer runs from the macOS package, and --image chooses an installed one. " + f"To run the image in Docker under emulation, which is much slower, pass --runtime docker.") + notes = [] + if kind == "version": + if value not in packages: + raise LauncherError(f"FastSurfer {value} is not installed as a macOS package; {install}.") + version = value + else: + versions = sorted(packages, key=parse_version) + if not versions: + raise LauncherError(f"FastSurfer is not installed. On a Mac with Apple silicon, it runs from the macOS " + f"package: {install}, or download it from {MACOS_PACKAGE_URL} (see {MACOS_DOC_URL}).") + version = versions[-1] + if len(versions) > 1 and not other_program: + raise LauncherError(f"Several FastSurfer macOS packages are installed: {', '.join(reversed(versions))}. " + f"Choose one with --image , e.g. --image {version}, the newest installed.") + if len(versions) > 1: + notes.append(("INFO", f"Several FastSurfer macOS packages are installed, this uses the newest, {version}; " + f"choose another with --image .")) + if newest is not None and parse_version(newest) > parse_version(version): + notes.append(("INFO", f"FastSurfer {newest} is available; {install}.")) + if not other_program: + notes.insert(0, ("INFO", f"Running FastSurfer {version} (the macOS package in {packages[version]}). For a " + f"study, keep this version with --image {version}.")) + return packages[version], notes + + +def macos_package_download(version: str | None) -> tuple[str, str, str]: + """The version, download URL and file name of the macOS package of a release (the newest if None).""" + import json + + url = f"{GITHUB_RELEASES}/tags/v{version}" if version else f"{GITHUB_RELEASES}/latest" + try: + with open_url(url) as response: + release = json.load(response) + except (OSError, ValueError) as error: + raise LauncherError(f"Could not find the macOS package{f' of FastSurfer {version}' if version else ''} on " + f"GitHub: {error}") from error + # the file name differs between releases, e.g. FastSurfer254-macos-darwin_arm64.pkg + assets = [asset for asset in release.get("assets", []) + if re.fullmatch(r"FastSurfer\w*-macos-darwin_arm64\.pkg", asset.get("name", ""))] + if not assets: + raise LauncherError(f"FastSurfer {release.get('tag_name', version)} has no macOS package.") + return release["tag_name"].lstrip("v"), assets[0]["browser_download_url"], assets[0]["name"] + + +def macos_command(home: Path, program: str, args: list[str]) -> list[str]: + """The command running `program` with the macOS package in `home`, set up as in the console it opens.""" + setup = 'source "$1" > /dev/null; shift; exec "$@"' + return ["/bin/bash", "-c", setup, "bash", str(home / MACOS_SETUP), program] + args + + def is_apple_silicon() -> bool: """Whether this is a Mac with Apple silicon, also when Python itself runs under Rosetta.""" if platform.system() != "Darwin": @@ -919,17 +1293,60 @@ def is_apple_silicon() -> bool: return out.stdout.strip() == "1" -def docker_memory_gib() -> float: - """The memory Docker's containers can use; on macOS and Windows, that of Docker's VM.""" +def is_wsl() -> bool: + """Whether this is Linux in WSL2 on Windows, whose kernel names Microsoft.""" + return platform.system() == "Linux" and "microsoft" in platform.release().lower() + + +def no_runtime_message(system: str, wsl: bool) -> str: + """What to install when neither Apptainer, Singularity nor Docker is, on this system.""" + if system == "Darwin": + return f"Docker is not installed. On an Intel Mac, FastSurfer runs in Docker Desktop, see {SETUP_URL}." + if wsl: + return (f"Docker is not available in this WSL distribution. Install Docker Desktop in Windows and switch on " + f"its WSL integration for this distribution (Settings > Resources > WSL integration), see {SETUP_URL}.") + return (f"Neither Apptainer, Singularity nor Docker is installed. On a cluster, Apptainer is often a module " + f"(module load apptainer); otherwise an administrator installs Apptainer or Docker, see {SETUP_URL}.") + + +def docker_problem(error: str, system: str, wsl: bool) -> str: + """What to do when `docker info` fails with this error, on this system.""" + lowered = error.lower() + desktop = system == "Darwin" or wsl + if "could not be found in this wsl" in lowered: + return (f"Docker Desktop's WSL integration is off for this distribution: switch it on under Settings > " + f"Resources > WSL integration, see {SETUP_URL}.") + if "permission denied" in lowered and not desktop: + return (f"Docker is installed, but this user may not use it: an administrator adds you to the docker group, " + f"then log out and back in, see {SETUP_URL}.") + if "cannot connect to the docker daemon" in lowered or "is the docker daemon running" in lowered: + if desktop: + return "Docker is installed, but not running: start Docker Desktop." + return "Docker is installed, but its service is not running: an administrator starts it." + reason = (error.strip().splitlines() or ["docker info failed"])[-1] + return f"Docker is installed, but does not work: {reason}" + + +def docker_info() -> tuple[float, set[str]]: + """ + The memory Docker's containers can use (on macOS and Windows, that of Docker's VM) and the runtimes Docker + knows, e.g. nvidia with the NVIDIA Container Toolkit; stops with what to do if Docker cannot be used. + """ + import json + try: - out = subprocess.run(["docker", "info", "--format", "{{.MemTotal}}"], + out = subprocess.run(["docker", "info", "--format", "{{.MemTotal}} {{json .Runtimes}}"], capture_output=True, text=True, timeout=60) except (OSError, subprocess.SubprocessError) as error: raise LauncherError(f"Could not ask Docker for its settings: {error}") from error if out.returncode != 0: - reason = (out.stderr.strip().splitlines() or ["docker info failed"])[-1] - raise LauncherError(f"Docker is installed, but not running: {reason}") - return int(out.stdout.strip()) / 2**30 + raise LauncherError(docker_problem(out.stderr or out.stdout, platform.system(), is_wsl())) + memory, _, runtimes = out.stdout.strip().partition(" ") + try: + names = set(json.loads(runtimes or "{}")) + except ValueError: + names = set() + return int(memory) / 2**30, names def _last_value(args: list[str], flag: str) -> str | None: @@ -992,10 +1409,8 @@ def build_image(command: list[str], anonymous: bool = False) -> bool: def stable_launcher() -> bytes | None: """The launcher on the stable branch, which knows the images of the newest release; None if unreachable.""" - import urllib.request - try: - with urllib.request.urlopen(LAUNCHER_URL, timeout=HUB_TIMEOUT) as response: + with open_url(LAUNCHER_URL) as response: return response.read() except OSError: return None @@ -1029,47 +1444,267 @@ def update_launcher() -> int: def make_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - prog="fastsurfer_container.py", + prog="fastsurfer", description=__doc__.split("\n\n")[0].strip(), epilog=( - "All other arguments are passed to run_fastsurfer.sh (or brun_fastsurfer.sh), see " - "https://deep-mi.org/FastSurfer/stable/scripts/RUN_FASTSURFER.html. Relative paths are " + "All other arguments are passed to run_fastsurfer.sh (or brun_fastsurfer.sh, long_fastsurfer.sh), " + "see https://deep-mi.org/FastSurfer/stable/scripts/RUN_FASTSURFER.html. Relative paths are " "relative to the current directory." ), allow_abbrev=False, ) parser.add_argument("--image", help="the image to run: a release tag such as cu132-v2.6.0 (to keep a study on one " - "image), a version such as 2.6.0 (the image type is chosen for this machine), a " - "Docker image (e.g. my_fastsurfer:cu132) or an Apptainer .sif file. Default: the " - "release image installed on this machine, or the newest release if there is none") + "image), a version such as 2.6.0 (the image type is chosen for this machine), rocm " + "for the ROCm image on an AMD GPU (experimental; used without --image when there is " + "no NVIDIA GPU), a Docker image (e.g. my_fastsurfer:cu132) or an Apptainer .sif file. " + "On a Mac with Apple silicon, the version of the macOS package (e.g. 2.6.0) or its " + "folder. Default: the release installed on this machine, or the newest release if " + "there is none") parser.add_argument("--update", action="store_true", help="download the newest release (or the one --image names) again, even if an image " - "is installed, and run it") + "is installed, and run it. On a Mac with Apple silicon, download the macOS package and " + "open the macOS Installer") parser.add_argument("--update_launcher", action="store_true", help="replace this launcher with the current one and stop") - parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker"], default="auto", - help="the container runtime (default: auto, the one that has the image, else Apptainer, " - "Singularity, then Docker)") + parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker", "macos"], default="auto", + help="what runs FastSurfer (default: auto, on a Mac with Apple silicon the macOS package, " + "else the container runtime that has the image, else Apptainer, Singularity, then " + "Docker)") parser.add_argument("--image_dir", type=Path, default=Path(os.environ.get("FASTSURFER_IMAGE_DIR", "~/my_singularity_images")).expanduser(), help="where Apptainer/Singularity images are kept (default: $FASTSURFER_IMAGE_DIR or " "~/my_singularity_images)") - parser.add_argument("--brun", action="store_true", help="run brun_fastsurfer.sh to process several subjects") + parser.add_argument("--exec", metavar="PROGRAM", + help="run another program of the image instead of run_fastsurfer.sh. It comes last: all " + "later arguments go to the program. brun_fastsurfer.sh (several subjects) and " + "long_fastsurfer.sh (longitudinal processing) take FastSurfer's flags, as " + "run_fastsurfer.sh does. Any other program, e.g. --exec nib-convert in.mgz out.nii.gz, " + "runs in the current directory, which is shared with the container, as are the " + "directories of absolute paths in its arguments") parser.add_argument("--runtime_args", default="", help="additional arguments for the container runtime, e.g. --runtime_args '--group-add 1001'") parser.add_argument("--print_only", action="store_true", help="print the commands instead of running them") return parser +def split_exec(argv: list[str]) -> tuple[list[str], list[str] | None]: + """ + The launcher's arguments up to `--exec ` (or `--exec=`), and the program's arguments + after it (None without --exec), so that its flags, e.g. --help, reach the program rather than the launcher. + """ + for k, arg in enumerate(argv): + if arg == "--exec": + return argv[:k + 2], argv[k + 2:] + if arg.startswith("--exec="): + return argv[:k + 1], argv[k + 1:] + return argv, None + + +def fastsurfer_script(program: str) -> str | None: + """The path in the image of a FastSurfer script --exec names, by name or path; None for other programs.""" + name = program.rsplit("/", 1)[-1] + script = SCRIPTS.get(name) + return script if script is not None and program in (name, script) else None + + +def macos_commands(options: argparse.Namespace, script: str | None, other: bool, fastsurfer_args: list[str], + info) -> list[list[str]]: + """ + The commands that run FastSurfer, or the program --exec names, with the macOS package on a Mac with + Apple silicon. With --update, they download the package and open the macOS Installer instead. + """ + kind, value = parse_image_spec(options.image) + hub = hub_tags() if kind in ("auto", "version") else None + if options.update: + version, url, name = macos_package_download(value if kind == "version" else None) + target = Path.home() / "Downloads" / name + info(f"Downloading the macOS package of FastSurfer {version} to {target}; the macOS Installer opens next. " + f"Once it is installed, run the same command again without --update.") + return [["curl", "-fL", "--progress-bar", "-o", str(target), url], ["open", str(target)]] + home, notes = decide_macos_package(kind, value, macos_packages(), newest_version(list(hub or []), "cpu"), + other) + if hub is not None: + stable = stable_launcher() + if stable is not None and stable != Path(__file__).resolve().read_bytes(): + info("A newer launcher is available, update it with --update_launcher.") + for level, note in notes: + info(note, level=level) + if other: + # it runs natively, in the current directory, so its paths need no care + return [macos_command(home, options.exec, fastsurfer_args)] + cwd = Path.cwd() + args, _ = plan_paths(fastsurfer_args + _license_args(fastsurfer_args, cwd), cwd, create=not options.print_only) + if "--fs_license" not in args and "--seg_only" not in args: + info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") + return [macos_command(home, str(home / script.rsplit("/", 1)[-1]), args)] + + +def container_commands(options: argparse.Namespace, script: str | None, other: bool, fastsurfer_args: list[str], + apple_silicon: bool, info) -> tuple[list[list[str]], str | None]: + """ + The commands that run FastSurfer, or the program --exec names, in its container image, and the digest + to record next to an Apptainer image they build (None if they build none). + """ + brun = script == BRUN_SCRIPT + # --update without FastSurfer's flags only downloads (or builds) the image + download_only = options.update and not other and not fastsurfer_args + installed = {name for name in ("apptainer", "singularity", "docker") if shutil.which(name)} + if not installed: + raise LauncherError(no_runtime_message(platform.system(), is_wsl())) + if options.runtime != "auto" and options.runtime not in installed: + raise LauncherError(f"{options.runtime} is not installed.") + if brun and not {"--subjects", "--subjects_list", "--subject_list", "--help"} & set(fastsurfer_args): + # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get + raise LauncherError("brun_fastsurfer.sh needs the subjects in --subjects_list or " + "--subjects = ...") + + cwd = Path.cwd() + if other: + args, mounts = fastsurfer_args, exec_mounts(fastsurfer_args, cwd) + else: + fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) + args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) + mounts = minimal_mounts(mounts) + if Path("/") in mounts: + raise LauncherError("The paths would share the whole file system with the container, keep the " + "data in a directory below /.") + if Path.home() in mounts: + info("This shares your home directory with the container. Settings in it may affect " + "FastSurfer, so keep the data in a subdirectory.") + if not other and not download_only and "--fs_license" not in args and "--seg_only" not in args: + info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") + + # the GPUs and the image type; another program's flags say nothing about them + kind, value = parse_image_spec(options.image) + if kind == "macos": + raise LauncherError(f"--image {value} is a macOS package, which runs only on a Mac with Apple silicon, " + f"without --runtime docker.") + gpus = query_nvidia_gpus() + request = [] if other else args + # whether the newest images support none of the GPUs, so they run on the CPU here + gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None + build = plan_image_build(kind, value, request, gpus, query_amd_gpus(), brun) + if kind == "rocm": + kind = "auto" + + # the image: one request to Docker Hub decides about versions + hub = None if kind in ("sif", "docker") else hub_tags() + if hub is not None and kind in ("auto", "version") and build.prefix != "cpu": + newest_release = newest_version(list(hub), "cpu") + if newest_release and f"{build.prefix}-v{newest_release}" not in hub: + raise LauncherError(f"FastSurfer {newest_release} has no {build.prefix} image, which means this " + f"launcher does not know the images of the newest release. Update it with " + f"--update_launcher, or download it again from {LAUNCHER_URL}.") + local_id = local_docker_id(value) if kind == "docker" else None + # the GPUs this run uses, to tell which installed images run here; none for a run on the CPU, and + # for AMD GPUs only the ROCm image of this launcher counts + used = [g for g in gpus if build.devices is None or g.index in build.devices] if build.gpu else [] + if build.amd or build.prefix.startswith("rocm"): + used = None + plan = decide_image(kind, value, build.prefix, options.runtime, installed, + local_docker_images() if "docker" in installed else None, + local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, + used, other_program=other or download_only) + runtime = plan.runtime + if hub is not None: + stable = stable_launcher() + if stable is not None and stable != Path(__file__).resolve().read_bytes(): + info("A newer launcher is available, update it with --update_launcher.") + + if runtime != "docker": + try: + version_text = subprocess.run([runtime, "--version"], capture_output=True, text=True, + timeout=30).stdout + except (OSError, subprocess.SubprocessError): + version_text = "" + problem = runtime_too_old(version_text) + if problem is not None: + raise LauncherError(problem) + + requested = value if kind == "version" else (plan.tag if kind in ("tag", "release") else None) + if requested is not None and hub is not None: + version = requested.rsplit("-v", 1)[-1] + note = age_note(version, release_versions(list(hub), "cpu"), gpu_unsupported and build.gpu) + if note is not None: + info(note[1], level=note[0]) + for level, note in plan.notes: + info(note, level=level) + for note in build.notes: + info(note) + + # another program runs in the current directory; FastSurfer's paths are absolute already + program, workdir = (options.exec, cwd) if other else (script, None) + extra = shlex.split(options.runtime_args) + commands = [] + if runtime == "docker": + user = f"{os.getuid()}:{os.getgid()}" + # official images are built for Intel/AMD only; an own image may be built for this Mac + emulate = apple_silicon and (kind != "docker" or value.startswith(DOCKER_REPO)) + if plan.pull: + commands.append(["docker", "pull"] + (["--platform", "linux/amd64"] if emulate else []) + [plan.image]) + # Apptainer passes input and terminal on itself, Docker only when asked: another program gets the + # input when it comes from a pipe or file, and the terminal when it runs in one. Input from a + # terminal without one, e.g. a job started in the background, would stop the job. + stdin_tty = sys.stdin is not None and sys.stdin.isatty() + interactive = other and not stdin_tty + tty = other and stdin_tty and sys.stdout.isatty() + commands.append(docker_command(plan.image, build.gpu, mounts, program, args, extra, user, emulate, + build.devices, workdir, interactive, build.amd, build.env, + amd_device_groups() if build.amd else None, tty)) + if not options.print_only: + memory, runtimes = docker_info() + if memory < MIN_DOCKER_MEMORY_GIB and not download_only: + if platform.system() == "Darwin": + where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " + "--memory 16)") + elif is_wsl(): + # Docker Desktop on WSL2 runs in the memory WSL2 has + where = ("of WSL2 in %UserProfile%\\.wslconfig (memory=16GB in the [wsl2] section), then run " + "wsl --shutdown") + else: + where = "for Docker" + info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " + f"15 GB. Increase the memory {where}.", level="WARNING") + # Docker on Linux reaches NVIDIA GPUs through the NVIDIA Container Toolkit, which Docker Desktop has + # built in; it registers the nvidia runtime or, in newer versions, only installs its hook + toolkit = "nvidia" in runtimes or any(shutil.which(name) for name in ("nvidia-container-runtime-hook", + "nvidia-ctk")) + if build.gpu and not build.amd and platform.system() == "Linux" and not is_wsl() and not toolkit: + info(f"Docker seems to lack the NVIDIA Container Toolkit, which it needs for the GPU. If Docker " + f"stops with 'could not select device driver', an administrator installs it (see {SETUP_URL}), " + f"or --device cpu runs FastSurfer on the CPU.", level="WARNING") + else: + if kind == "sif" and not Path(plan.image).is_file(): + raise LauncherError(f"--image {value}: no such file.") + if plan.build_from is not None: + commands.append([runtime, "build", plan.image, plan.build_from]) + commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, program, args, extra, + build.devices, workdir, build.amd, build.env)) + if download_only: + commands = commands[:-1] + if not commands: + raise LauncherError(f"--update has nothing to download for {plan.image}.") + return commands, plan.record + + def main(argv: list[str] | None = None) -> int: parser = make_parser() + argv, program_args = split_exec(sys.argv[1:] if argv is None else argv) options, fastsurfer_args = parser.parse_known_args(argv) if options.update_launcher: return update_launcher() - if not fastsurfer_args: + if options.exec is not None: + if fastsurfer_args: + parser.error(f"unrecognized arguments before --exec: {' '.join(fastsurfer_args)}") + fastsurfer_args = program_args or [] + elif not fastsurfer_args and not options.update: + # --update alone downloads (and on a Mac with Apple silicon installs) the newest release parser.print_help() return 1 + # the FastSurfer script to run, None for another program, which has its own flags + script = RUN_SCRIPT if options.exec is None else fastsurfer_script(options.exec) + other = script is None def info(message: str, level: str = "INFO") -> None: print(f"{level}: {message}", file=sys.stderr) @@ -1077,119 +1712,19 @@ def info(message: str, level: str = "INFO") -> None: try: if os.name != "posix": raise LauncherError("Run this in WSL on Windows, where Docker's paths are Linux paths.") - # the images are built for Intel/AMD processors, which a Mac with Apple silicon only emulates + # the images are built for Intel/AMD processors, which a Mac with Apple silicon only emulates, so + # it runs the macOS package instead, unless Docker is asked for apple_silicon = is_apple_silicon() - package_hint = (f"install the FastSurfer package for macOS instead, {MACOS_PACKAGE_URL} (see {MACOS_DOC_URL}). " - f"It runs natively and uses the Mac's GPU: in the console it opens, call run_fastsurfer.sh " - f"with the same flags.") - if apple_silicon and options.runtime != "docker": - raise LauncherError(f"On a Mac with Apple silicon, {package_hint} To run the Intel image in Docker " - f"under emulation anyway, which is much slower, pass --runtime docker.") - if apple_silicon: - info(f"Docker on a Mac with Apple silicon emulates the Intel image, which is much slower than the " - f"macOS package and cannot use the GPU. Better {package_hint}", level="WARNING") - installed = {name for name in ("apptainer", "singularity", "docker") if shutil.which(name)} - if not installed: - raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") - if options.runtime != "auto" and options.runtime not in installed: - raise LauncherError(f"{options.runtime} is not installed.") - if options.brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): - # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get - raise LauncherError("--brun needs the subjects in --subjects_list or --subjects = ...") - - cwd = Path.cwd() - fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) - args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) - mounts = minimal_mounts(mounts) - if Path("/") in mounts: - raise LauncherError("The paths would share the whole file system with the container, keep the " - "data in a directory below /.") - if Path.home() in mounts: - info("This shares your home directory with the container. Settings in it may affect " - "FastSurfer, so keep the data in a subdirectory.") - if "--fs_license" not in args and "--seg_only" not in args: - info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") - - # the GPUs and the image type - kind, value = parse_image_spec(options.image) - gpus = query_nvidia_gpus() - # whether the newest images support none of the GPUs, so they run on the CPU here - gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None - wants_gpu = gpu_request(args, options.brun)[0] != "cpu" and bool(gpus) - if kind in ("auto", "version"): - build = plan_build(args, gpus, options.brun) - elif kind in ("tag", "release"): - build = plan_build_for_type(args, gpus, options.brun, RELEASE_TAG.fullmatch(value).group(1)) - else: - build = Build("custom", wants_gpu) - - # the image: one request to Docker Hub decides about versions - hub = None if kind in ("sif", "docker") else hub_tags() - if hub is not None and kind in ("auto", "version") and build.prefix != "cpu": - newest_release = newest_version(list(hub), "cpu") - if newest_release and f"{build.prefix}-v{newest_release}" not in hub: - raise LauncherError(f"FastSurfer {newest_release} has no {build.prefix} image, which means this " - f"launcher does not know the images of the newest release. Update it with " - f"--update_launcher, or download it again from {LAUNCHER_URL}.") - local_id = local_docker_id(value) if kind == "docker" else None - # the GPUs this run uses, to tell which installed images run here; none for a run on the CPU - used = [g for g in gpus if build.devices is None or g.index in build.devices] if build.gpu else [] - plan = decide_image(kind, value, build.prefix, options.runtime, installed, - local_docker_images() if "docker" in installed else None, - local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, - used) - runtime = plan.runtime - if hub is not None: - stable = stable_launcher() - if stable is not None and stable != Path(__file__).resolve().read_bytes(): - info("A newer launcher is available, update it with --update_launcher.") - - if runtime != "docker": - try: - version_text = subprocess.run([runtime, "--version"], capture_output=True, text=True, - timeout=30).stdout - except (OSError, subprocess.SubprocessError): - version_text = "" - problem = runtime_too_old(version_text) - if problem is not None: - raise LauncherError(problem) - - requested = value if kind == "version" else (plan.tag if kind in ("tag", "release") else None) - if requested is not None and hub is not None: - version = requested.rsplit("-v", 1)[-1] - note = age_note(version, release_versions(list(hub), "cpu"), gpu_unsupported and build.gpu) - if note is not None: - info(note[1], level=note[0]) - for level, note in plan.notes: - info(note, level=level) - for note in build.notes: - info(note) - - script = "brun_fastsurfer.sh" if options.brun else "run_fastsurfer.sh" - extra = shlex.split(options.runtime_args) - commands = [] - if runtime == "docker": - user = f"{os.getuid()}:{os.getgid()}" - # official images are built for Intel/AMD only; an own image may be built for this Mac - emulate = apple_silicon and (kind != "docker" or value.startswith(DOCKER_REPO)) - if plan.pull: - commands.append(["docker", "pull"] + (["--platform", "linux/amd64"] if emulate else []) + [plan.image]) - commands.append(docker_command(plan.image, build.gpu, mounts, script, args, extra, user, emulate, - build.devices)) - if not options.print_only: - memory = docker_memory_gib() - if memory < MIN_DOCKER_MEMORY_GIB: - where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " - "--memory 16)") if platform.system() == "Darwin" else "for Docker" - info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " - f"15 GB. Increase the memory {where}.", level="WARNING") + if options.runtime == "macos" and not apple_silicon: + raise LauncherError("The macOS package of FastSurfer is for Macs with Apple silicon.") + if apple_silicon and options.runtime in ("auto", "macos"): + commands, record = macos_commands(options, script, other, fastsurfer_args, info), None else: - if kind == "sif" and not Path(plan.image).is_file(): - raise LauncherError(f"--image {value}: no such file.") - if plan.build_from is not None: - commands.append([runtime, "build", plan.image, plan.build_from]) - commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, script, args, extra, - build.devices)) + if apple_silicon: + info("Docker on a Mac with Apple silicon emulates the Intel image, which is much slower than the " + "macOS package and cannot use the GPU. Without --runtime docker, the launcher runs the package.", + level="WARNING") + commands, record = container_commands(options, script, other, fastsurfer_args, apple_silicon, info) except LauncherError as error: print(f"ERROR: {error}", file=sys.stderr) return 1 @@ -1198,7 +1733,9 @@ def info(message: str, level: str = "INFO") -> None: for command in commands: print(shlex.join(command)) return 0 - for command in commands[:-1]: + # --update alone runs only the download steps, otherwise the last command is FastSurfer (or --exec) + download_only = options.update and options.exec is None and not fastsurfer_args + for command in commands if download_only else commands[:-1]: if command[1] == "build": info(f"Building the image once: {shlex.join(command)}") # the official images are public, so their download needs no login @@ -1211,14 +1748,17 @@ def info(message: str, level: str = "INFO") -> None: "`apptainer registry list` and $APPTAINER_DOCKER_USERNAME.", file=sys.stderr) return 1 - record = Path(command[2]).with_name(Path(command[2]).name + ".digest") - if plan.record: - record.write_text(plan.record + "\n") - elif record.exists(): - record.unlink() + digest_file = Path(command[2]).with_name(Path(command[2]).name + ".digest") + if record: + digest_file.write_text(record + "\n") + elif digest_file.exists(): + digest_file.unlink() elif subprocess.run(command).returncode != 0: print(f"ERROR: {shlex.join(command)} failed.", file=sys.stderr) return 1 + if download_only: + info("Done. Run FastSurfer with the same command and its flags, without --update.") + return 0 info(f"Running: {shlex.join(commands[-1])}") sys.stdout.flush() sys.stderr.flush()