From 9b35ede84951ce292a63a836fab30df72549b5c0 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Sun, 16 Aug 2026 11:27:38 -0300 Subject: [PATCH 01/32] Add basic support to Apple container --- nf_core/__main__.py | 2 +- nf_core/pipeline-template/README.md | 1 + nf_core/pipeline-template/docs/usage.md | 4 ++- nf_core/pipeline-template/nextflow.config | 35 +++++++++++++++++------ nf_core/pipelines/containers_utils.py | 1 + nf_core/pipelines/download/docker.py | 2 +- nf_core/pipelines/download/download.py | 22 +++++++------- nf_core/pipelines/download/singularity.py | 1 + 8 files changed, 45 insertions(+), 23 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index bdf60b8cda..c77eab1738 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -342,7 +342,7 @@ def command_pipelines_lint( @click.option( "-s", "--container-system", - type=click.Choice(["none", "singularity", "docker", "apptainer"]), + type=click.Choice(["none", "singularity", "docker", "apptainer", "appleContainer"]), help="Download container images of required software.", ) @click.option( diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 57908aa578..952e277760 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -26,6 +26,7 @@ [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) +[![run with apple container](https://img.shields.io/badge/run%20with-apple%20container-000000.svg?labelColor=000000&logo=apple)](https://github.com/apple/container) [![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/{{ name }}) {% endif -%} diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index ca7e7748fd..255a2cfc58 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -164,10 +164,12 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://charliecloud.io/) - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) +- `appleContainer` + - A generic configuration profile to be used with [Apple container](https://github.com/apple/container) on macOS (Apple silicon) - `wave` - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow `24.03.0-edge` or later). - `conda` - - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. + - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, Apptainer, or Apple container. ### `-resume` diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 34806e4729..7ce397763e 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -107,8 +107,9 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - conda.channels = ['conda-forge', 'bioconda'] apptainer.enabled = false + appleContainer.enabled = false + conda.channels = ['conda-forge', 'bioconda'] } mamba { conda.enabled = true @@ -119,6 +120,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } docker { docker.enabled = true @@ -128,6 +130,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false docker.runOptions = '-u $(id -u):$(id -g)' } arm64 { @@ -153,6 +156,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } podman { podman.enabled = true @@ -162,6 +166,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } shifter { shifter.enabled = true @@ -171,6 +176,7 @@ profiles { podman.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } charliecloud { charliecloud.enabled = true @@ -180,6 +186,7 @@ profiles { podman.enabled = false shifter.enabled = false apptainer.enabled = false + appleContainer.enabled = false } apptainer { apptainer.enabled = true @@ -190,6 +197,17 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + appleContainer.enabled = false + } + appleContainer { + appleContainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } wave { apptainer.ociAutoPull = true @@ -236,14 +254,15 @@ includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !pa // includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/pipeline/{{ short_name }}.config" : "/dev/null" {%- endif %} -// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile -// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled +// Set default registry for Apptainer, Docker, Podman, Charliecloud, Singularity and Apple container independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity / Apple container are enabled // Set to your registry if you have a mirror of containers -apptainer.registry = 'quay.io' -docker.registry = 'quay.io' -podman.registry = 'quay.io' -singularity.registry = 'quay.io' -charliecloud.registry = 'quay.io' +apptainer.registry = 'quay.io' +appleContainer.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' +charliecloud.registry = 'quay.io' {% if igenomes -%} // Load igenomes.config if required diff --git a/nf_core/pipelines/containers_utils.py b/nf_core/pipelines/containers_utils.py index d7de4fe73c..9a21fd7c1b 100644 --- a/nf_core/pipelines/containers_utils.py +++ b/nf_core/pipelines/containers_utils.py @@ -17,6 +17,7 @@ "singularity_oras_arm64": ["singularity", "linux/arm64", "name"], "singularity_https_amd64": ["singularity", "linux/amd64", "https"], "singularity_https_arm64": ["singularity", "linux/arm64", "https"], + "appleContainer_arm64": ["docker", "linux/arm64", "name"], "conda_lock_files_amd64": ["conda", "linux/amd64", "lock_file"], "conda_lock_files_arm64": ["conda", "linux/arm64", "lock_file"], } diff --git a/nf_core/pipelines/download/docker.py b/nf_core/pipelines/download/docker.py index 247a1579ba..6c3bce6e40 100644 --- a/nf_core/pipelines/download/docker.py +++ b/nf_core/pipelines/download/docker.py @@ -100,7 +100,7 @@ def gather_registries(self, workflow_directory: Path) -> set[str]: set[str]: The set of registries to use for the container download """ registry_set = self.base_registry_set.copy() - configured_registry_keys = ["docker.registry", "podman.registry"] + configured_registry_keys = ["appleContainer.registry", "docker.registry", "podman.registry"] # Add the registries defined in the workflow config registry_set |= self.gather_config_registries( diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index ef3b84dfcf..77b6652409 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -43,8 +43,6 @@ force_terminal=nf_core.utils.rich_force_colors(), ) -SINGULARITY_SYSTEMS = ["singularity", "apptainer"] - class DownloadWorkflow: """Downloads a nf-core workflow from GitHub to the local file system. @@ -87,12 +85,12 @@ def __init__( ): # Verify that the flags provided make sense together if ( - container_system == "docker" + container_system in ("docker", "appleContainer") and container_cache_utilisation != "copy" and container_cache_utilisation is not None ): raise DownloadError( - "Only the 'copy' option for --container-cache-utilisation is supported for Docker images. " + "Only the 'copy' option for --container-cache-utilisation is supported for Docker/Apple container images. " ) self._pipeline = pipeline @@ -251,7 +249,7 @@ def download_workflow(self) -> None: ] if self.container_system: summary_log.append(f"Container library: '{', '.join(self.container_library)}'") - if self.container_system in SINGULARITY_SYSTEMS and os.environ.get(SINGULARITY_CACHE_DIR_ENV_VAR) is not None: + if self.container_system in ("singularity", "apptainer") and os.environ.get(SINGULARITY_CACHE_DIR_ENV_VAR) is not None: summary_log.append( f"Using [blue]{SINGULARITY_CACHE_DIR_ENV_VAR}[/]': {os.environ[SINGULARITY_CACHE_DIR_ENV_VAR]}'" ) @@ -333,7 +331,7 @@ def download_workflow_static(self) -> None: raise DownloadError("Error editing pipeline config file to use local configs!") from e # Collect all required container images - if self.container_system in {"singularity", "docker", "apptainer"}: + if self.container_system in ("singularity", "apptainer", "docker", "appleContainer"): workflow_directory = self.outdir / revision_dirname self.find_container_images(workflow_directory, revision) @@ -369,11 +367,11 @@ def download_workflow_platform(self, location: Path | None = None) -> None: self.workflow_repo.bare_clone(self.output_filename) # extract the required containers - if self.container_system in {"singularity", "docker", "apptainer"}: + if self.container_system in ("singularity", "apptainer", "docker", "appleContainer"): for revision, commit in self.wf_sha.items(): # Checkout the repo in the current revision self.workflow_repo.checkout(commit) - # Collect all required singularity images + # Collect all required container images workflow_directory = self.workflow_repo.access() self.find_container_images(workflow_directory, revision) @@ -505,7 +503,7 @@ def prompt_container_download(self) -> None: stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container_system = questionary.select( "Download software container images:", - choices=["none", "singularity", "docker", "apptainer"], + choices=["none", "singularity", "docker", "apptainer", "appleContainer"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -515,7 +513,7 @@ def setup_container_fetcher(self) -> None: """ assert self.outdir is not None # mypy try: - if self.container_system in SINGULARITY_SYSTEMS: + if self.container_system in ("singularity", "apptainer"): self.container_fetcher = SingularityFetcher( outdir=self.outdir, container_library=self.container_library, @@ -526,7 +524,7 @@ def setup_container_fetcher(self) -> None: hide_progress=self.hide_progress, container_system=self.container_system, ) - elif self.container_system == "docker": + elif self.container_system in ("docker", "appleContainer"): self.container_fetcher = DockerFetcher( outdir=self.outdir, registry_set=self.registry_set, @@ -559,7 +557,7 @@ def prompt_compression_type(self) -> None: stderr.print( "\nIf transferring the downloaded files to another system, it can be convenient to have everything compressed in a single file." ) - if self.container_system in SINGULARITY_SYSTEMS: + if self.container_system in ("singularity", "apptainer"): stderr.print( "[bold]This is [italic]not[/] recommended when downloading Singularity/Apptainer images, as it can take a long time and saves very little space." ) diff --git a/nf_core/pipelines/download/singularity.py b/nf_core/pipelines/download/singularity.py index 98d4b5981a..3f57912953 100644 --- a/nf_core/pipelines/download/singularity.py +++ b/nf_core/pipelines/download/singularity.py @@ -228,6 +228,7 @@ def gather_registries(self, workflow_directory: Path) -> set[str]: # Select registries defined in pipeline config configured_registry_keys = [ + "appleContainer.registry", "apptainer.registry", "docker.registry", "podman.registry", From 01219aef3954632452cbe5470de3fa4e53177d43 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Sun, 16 Aug 2026 11:55:31 -0300 Subject: [PATCH 02/32] Add support to Apple container downloading images --- nf_core/pipelines/download/apple_container.py | 147 +++++++++++++ nf_core/pipelines/download/download.py | 11 +- .../load_scripts/appleContainer-load.sh | 29 +++ .../download/test_apple_container.py | 198 ++++++++++++++++++ 4 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 nf_core/pipelines/download/apple_container.py create mode 100644 nf_core/pipelines/download/load_scripts/appleContainer-load.sh create mode 100644 tests/pipelines/download/test_apple_container.py diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py new file mode 100644 index 0000000000..b119ca0fb9 --- /dev/null +++ b/nf_core/pipelines/download/apple_container.py @@ -0,0 +1,147 @@ +"""Apple Container fetcher for nf-core download. + +Apple Container (https://github.com/apple/container) is a lightweight +container runtime for macOS on Apple Silicon. It uses standard OCI/Docker +images and provides its own CLI (``container``) with native ``pull`` and +``save`` commands for offline bundling. + +Images are pulled and saved using the ``container`` CLI directly — no +Docker installation is required. Loading images on the target machine +uses ``container image load``. +""" + +import logging +import shutil +from collections.abc import Iterable +from pathlib import Path + +import rich.console + +import nf_core.utils +from nf_core.pipelines.download.container_fetcher import ContainerFetcher +from nf_core.pipelines.download.docker import DockerFetcher +from nf_core.pipelines.download.utils import copy_container_load_scripts +from nf_core.utils import ContainerRegistryUrls + +log = logging.getLogger(__name__) +stderr = rich.console.Console( + stderr=True, + highlight=False, + force_terminal=nf_core.utils.rich_force_colors(), +) + + +class AppleContainerFetcher(DockerFetcher): + """ + Fetcher for Apple Container images. + + Subclasses :class:`DockerFetcher` — uses the native Apple Container CLI + (``container image pull`` / ``container image save``) instead of Docker. + The cleanup step writes an Apple-Container-specific load script. + """ + + def __init__( + self, + outdir: Path, + container_library: Iterable[str], + registry_set: Iterable[str], + parallel: int = 4, + hide_progress: bool = False, + ): + super().__init__( + outdir=outdir, + container_library=container_library, + registry_set=registry_set, + parallel=parallel, + hide_progress=hide_progress, + ) + # Override the container output directory name + self._container_output_dir = outdir / "apple-container-images" + + def check_and_set_implementation(self) -> None: + """ + Check if Apple Container CLI is installed and set the implementation. + """ + container_binary = shutil.which("container") + if not container_binary: + raise OSError( + "Apple Container CLI ('container') is needed to pull images, " + "but it is not installed or not in $PATH.\n" + "See: https://github.com/apple/container" + ) + + self.implementation = "container" + + def construct_pull_command(self, address: str) -> list[str]: + """ + Construct the command to pull an image using Apple Container CLI. + + Args: + address (str): The address of the container to pull. + """ + pull_command = ["container", "image", "pull", "--platform", "linux/arm64", address] + log.debug(f"Apple Container command: {' '.join(pull_command)}") + return pull_command + + def construct_save_command(self, output_path: Path, address: str) -> list[str]: + """ + Construct the command to save an image using Apple Container CLI. + + Args: + output_path (Path): The path to save the container image. + address (str): The address of the container to save. + """ + save_command = [ + "container", + "image", + "save", + "--platform", + "linux/arm64", + "--output", + str(output_path), + address, + ] + log.debug(f"Apple Container command: {' '.join(save_command)}") + return save_command + + def gather_registries(self, workflow_directory: Path) -> set[str]: + """ + Gather the registries for Apple Container downloads. + + Checks ``appleContainer.registry``, ``docker.registry``, and + ``podman.registry`` keys from the workflow configuration. + """ + registry_set = self.base_registry_set.copy() + configured_registry_keys = ["appleContainer.registry", "docker.registry", "podman.registry"] + + registry_set |= self.gather_config_registries( + workflow_directory, + configured_registry_keys, + ) + + # Add the Seqera Docker container registry + registry_set.add(ContainerRegistryUrls.SEQERA_DOCKER.value) + return registry_set + + def cleanup(self) -> None: + """ + Write the Apple Container load message (skipping the Docker-specific one). + """ + # Call the grandparent cleanup directly to skip DockerFetcher.cleanup() + ContainerFetcher.cleanup(self) + self._write_apple_container_load_message() + + def _write_apple_container_load_message(self) -> None: + """ + Inform the user how to load downloaded images into Apple Container. + """ + img_dir = self.get_container_output_dir() + apple_load_script, _ = copy_container_load_scripts("appleContainer", img_dir) + indent = " " + stderr.print( + "\n" + f"{indent}Downloaded container images written to [magenta]'{img_dir}'[/].\n" + f"{indent}After copying the pipeline and images to the target macOS machine, run\n\n" + f"{indent}{indent}[green]./{apple_load_script}[/]\n\n" + f"{indent}inside [magenta]'{img_dir}'[/] to load the images into Apple Container.\n" + ) diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index 77b6652409..61d4ae4b91 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -22,6 +22,7 @@ import nf_core.utils from nf_core.github_api import gh_api from nf_core.pipelines.download.container_fetcher import ContainerFetcher +from nf_core.pipelines.download.apple_container import AppleContainerFetcher from nf_core.pipelines.download.docker import DockerFetcher from nf_core.pipelines.download.singularity import SINGULARITY_CACHE_DIR_ENV_VAR, SingularityFetcher from nf_core.pipelines.download.utils import DownloadError @@ -524,7 +525,15 @@ def setup_container_fetcher(self) -> None: hide_progress=self.hide_progress, container_system=self.container_system, ) - elif self.container_system in ("docker", "appleContainer"): + elif self.container_system == "appleContainer": + self.container_fetcher = AppleContainerFetcher( + outdir=self.outdir, + registry_set=self.registry_set, + container_library=self.container_library, + parallel=self.parallel, + hide_progress=self.hide_progress, + ) + elif self.container_system == "docker": self.container_fetcher = DockerFetcher( outdir=self.outdir, registry_set=self.registry_set, diff --git a/nf_core/pipelines/download/load_scripts/appleContainer-load.sh b/nf_core/pipelines/download/load_scripts/appleContainer-load.sh new file mode 100644 index 0000000000..5621349210 --- /dev/null +++ b/nf_core/pipelines/download/load_scripts/appleContainer-load.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail # Ensure that the script exits as early as possible + +LOGFILE="appleContainer-load.log" + +# Clear log +> "$LOGFILE" + +if ! command -v container &> /dev/null +then + echo "Error: Apple Container CLI ('container') is not installed. Please install it to continue." >&2 + echo "See: https://github.com/apple/container" >&2 + exit 1 +fi + +echo "Loading tar archives into Apple Container" +for tarfile in $(ls -1 *.tar); do + if output=$(container load < "$tarfile" 2>&1); then + echo "SUCCESS: $tarfile" + echo "SUCCESS: $tarfile" >> "$LOGFILE" + echo "$output" >> "$LOGFILE" + echo "----------------------------------------------------------------" >> "$LOGFILE" + else + echo "ERROR: $tarfile" + echo "ERROR: $tarfile" >> "$LOGFILE" + echo "$output" >> "$LOGFILE" + echo "----------------------------------------------------------------" >> "$LOGFILE" + fi +done diff --git a/tests/pipelines/download/test_apple_container.py b/tests/pipelines/download/test_apple_container.py new file mode 100644 index 0000000000..e554e637b2 --- /dev/null +++ b/tests/pipelines/download/test_apple_container.py @@ -0,0 +1,198 @@ +"""Tests for the Apple Container fetcher in nf-core download.""" + +import os +import shutil +import unittest +from contextlib import redirect_stderr +from io import StringIO +from pathlib import Path +from unittest import mock + +import pytest + +from nf_core.pipelines.download.apple_container import AppleContainerFetcher + +from ...utils import with_temporary_folder + + +class AppleContainerTest(unittest.TestCase): + @pytest.fixture(autouse=True) + def use_caplog(self, caplog): + self._caplog = caplog + + # + # Test for AppleContainerFetcher initialization and output directory + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_output_dir(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + assert fetcher.get_container_output_dir() == tmp_path / "apple-container-images" + + # + # Test for AppleContainerFetcher._write_apple_container_load_message + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_write_load_message(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + img_dir = fetcher.get_container_output_dir() + img_dir.mkdir() + with redirect_stderr(StringIO()) as f: + fetcher._write_apple_container_load_message() + + output = f.getvalue() + # Check that the message references correct scripts + # Rich may insert hard line breaks in long paths, so strip all whitespace for path check + assert "apple-container-images" in output.replace("\n", "").replace(" ", "") + assert "appleContainer-load.sh" in output + + # Check that the script was written and is executable + assert (img_dir / "appleContainer-load.sh").exists() + assert os.access(img_dir / "appleContainer-load.sh", os.X_OK) + + # + # Test that cleanup calls the Apple Container message, not the Docker one + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_cleanup(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + img_dir = fetcher.get_container_output_dir() + img_dir.mkdir() + with redirect_stderr(StringIO()) as f: + fetcher.cleanup() + + output = f.getvalue() + # Should mention Apple Container, not Docker/podman + assert "appleContainer-load.sh" in output + # Should NOT have the Docker-specific "podman-load.sh (experimental)" wording + assert "experimental" not in output + assert "podman" not in output + + # + # Test that gather_registries includes appleContainer.registry key + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + @mock.patch("nf_core.pipelines.download.container_fetcher.ContainerFetcher.gather_config_registries") + def test_apple_container_gather_registries(self, tmp_path, mock_gather_config): + tmp_path = Path(tmp_path) + mock_gather_config.return_value = {"quay.io"} + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=["docker.io"], + ) + registries = fetcher.gather_registries(tmp_path) + # Should have called gather_config_registries with the apple container key + mock_gather_config.assert_called_once_with( + tmp_path, + ["appleContainer.registry", "docker.registry", "podman.registry"], + ) + assert "quay.io" in registries + assert "docker.io" in registries + + # + # Test that check_and_set_implementation finds the container binary + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_check_implementation(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + assert fetcher.implementation == "container" + + # + # Test that check_and_set_implementation raises when container CLI is missing + # + @with_temporary_folder + @mock.patch("shutil.which", return_value=None) + def test_apple_container_missing_cli_raises(self, tmp_path, mock_which): + tmp_path = Path(tmp_path) + with pytest.raises(OSError, match="Apple Container CLI"): + AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + + # + # Test that construct_pull_command produces the correct command + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_pull_command(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + cmd = fetcher.construct_pull_command("quay.io/biocontainers/fastqc:0.12.1") + assert cmd == ["container", "image", "pull", "--platform", "linux/arm64", "quay.io/biocontainers/fastqc:0.12.1"] + + # + # Test that construct_save_command produces the correct command + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_save_command(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + output_path = tmp_path / "image.tar" + cmd = fetcher.construct_save_command(output_path, "quay.io/biocontainers/fastqc:0.12.1") + assert cmd == [ + "container", + "image", + "save", + "--platform", + "linux/arm64", + "--output", + str(output_path), + "quay.io/biocontainers/fastqc:0.12.1", + ] From a14bd272dd45dca829a63117611278b88c4837f3 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Sun, 16 Aug 2026 12:08:29 -0300 Subject: [PATCH 03/32] Finishing touches to the Apple container support - Add template_features.yml toggle for Apple Container in the template features - Add integration tests --- .../pipelines/create/template_features.yml | 19 ++++++++++ .../mock_pipeline_containers/nextflow.config | 15 ++++++-- .../appleContainer_containers.json | 11 ++++++ tests/pipelines/download/test_download.py | 36 +++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 tests/data/mock_pipeline_containers/per_profile_output/appleContainer_containers.json diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index eedd517933..733ea326e6 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -391,6 +391,25 @@ configurations: custom_pipelines: True default: False + apple_container: + skippable_paths: False + short_description: "Use Apple Container" + description: "Add Apple Container support to the pipeline" + help_text: | + This will add an `appleContainer` profile to the pipeline configuration, enabling + native container execution on macOS with Apple Silicon via the + [Apple Container](https://github.com/apple/container) runtime. + + Apple Container uses standard OCI/Docker images and provides its own CLI + (`container`) with native `pull` and `save` commands. No Docker Desktop + installation is required. + + The `nf-core pipelines download` command will also support downloading + images for offline use with `--container-system appleContainer`. + nfcore_pipelines: True + custom_pipelines: True + default: true + development_environments: name: "Development Environments" features: diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index d07c04a405..305edfb5c1 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -149,6 +149,16 @@ profiles { shifter.enabled = false charliecloud.enabled = false } + appleContainer { + appleContainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + } wave { apptainer.ociAutoPull = true singularity.ociAutoPull = true @@ -167,9 +177,10 @@ profiles { -// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile -// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled +// Set default registry for Apptainer, Docker, Podman, Charliecloud, Singularity and Apple Container independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity / Apple Container are enabled // Set to your registry if you have a mirror of containers +appleContainer.registry = 'quay.io' apptainer.registry = 'quay.io' docker.registry = 'quay.io' podman.registry = 'quay.io' diff --git a/tests/data/mock_pipeline_containers/per_profile_output/appleContainer_containers.json b/tests/data/mock_pipeline_containers/per_profile_output/appleContainer_containers.json new file mode 100644 index 0000000000..64ca108646 --- /dev/null +++ b/tests/data/mock_pipeline_containers/per_profile_output/appleContainer_containers.json @@ -0,0 +1,11 @@ +{ + "MOCK_DOCKER_SINGLE_QUAY_IO": "quay.io/biocontainers/singlequay:1.9--pyh9f0ad1d_0", + "MOCK_DSL2_APPTAINER_VAR1": "quay.io/biocontainers/dsltwoapptainervarone:1.1.0--py38h7be5676_2", + "MOCK_DSL2_APPTAINER_VAR2": "quay.io/biocontainers/dsltwoapptainervartwo:1.1.0--hdfd78af_0", + "MOCK_DSL2_CURRENT": "quay.io/biocontainers/dsltwocurrent:1.2.1--pyhdfd78af_0", + "MOCK_DSL2_CURRENT_INVERTED": "quay.io/biocontainers/dsltwocurrentinv:3.3.2--h1b792b2_1", + "MOCK_DSL2_OLD": "quay.io/biocontainers/dsltwoold:0.23.0--0", + "MOCK_SEQERA_CONTAINER_HTTP": "community.wave.seqera.io/library/coreutils:9.5--ae99c88a9b28c264", + "MOCK_SEQERA_CONTAINER_ORAS": "community.wave.seqera.io/library/umi-transfer:1.0.0--d30e8812ea280fa1", + "MOCK_SEQERA_CONTAINER_ORAS_MULLED": "community.wave.seqera.io/library/umi-transfer_umicollapse:3298d4f1b49e33bd" +} diff --git a/tests/pipelines/download/test_download.py b/tests/pipelines/download/test_download.py index 092817b067..d1b4639db7 100644 --- a/tests/pipelines/download/test_download.py +++ b/tests/pipelines/download/test_download.py @@ -353,6 +353,42 @@ def test_containers_pipeline_docker(self, tmp_path, mock_fetch_wf_config): f"Containers that should've been found: {ref_container_strs}" ) + # + # Test that `find_container_images` (uses `nextflow inspect`) fetches the correct Apple Container images + # + @pytest.mark.skipif( + shutil.which("nextflow") is None or not check_nextflow_version(NF_INSPECT_MIN_NF_VERSION), + reason=f"Can't run test that requires Nextflow >= {NF_INSPECT_MIN_NF_VERSION} to run if not installed.", + ) + @with_temporary_folder + @mock.patch("nf_core.utils.fetch_wf_config") + def test_containers_pipeline_apple_container(self, tmp_path, mock_fetch_wf_config): + tmp_path = Path(tmp_path) + assert check_nextflow_version(NF_INSPECT_MIN_NF_VERSION) is True + + # Set up test + container_system = "appleContainer" + mock_pipeline_dir = TEST_DATA_DIR / "mock_pipeline_containers" + refererence_json_dir = mock_pipeline_dir / "per_profile_output" + # Check that `-profile appleContainer` produces the same output as the reference + download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path, container_system=container_system) + mock_fetch_wf_config.return_value = {} + + # NF >= 26.04 rejects old-style if/else container directives (mock_dsl2_old uses them) + entrypoint = "main_passing_test.nf" + with self._strict_syntax_ctx(match="downgrade to Nextflow"): + download_obj.find_container_images(mock_pipeline_dir, "dummy-revision", entrypoint=entrypoint) + + if not self.nf_strict_syntax: + found_containers = set(download_obj.containers) + with open(refererence_json_dir / f"{container_system}_containers.json") as fh: + ref_containers = json.load(fh) + ref_container_strs = set(ref_containers.values()) + assert found_containers == ref_container_strs, ( + f"Containers found in pipeline by `nextflow inspect`: {found_containers}\n" + f"Containers that should've been found: {ref_container_strs}" + ) + @mock.patch("nf_core.pipelines.download.download.run_cmd") @mock.patch("nf_core.pipelines.list.Workflows.get_remote_workflows") def test_find_container_images_retries_with_outdir_on_missing_param_error( From 2c8137aebd992b73bf1b00bbb0a3efbb36fd5181 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sun, 16 Aug 2026 15:12:09 +0000 Subject: [PATCH 04/32] [automated] Fix code linting --- nf_core/pipelines/download/download.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index 61d4ae4b91..5477e9a670 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -21,8 +21,8 @@ import nf_core.pipelines.list import nf_core.utils from nf_core.github_api import gh_api -from nf_core.pipelines.download.container_fetcher import ContainerFetcher from nf_core.pipelines.download.apple_container import AppleContainerFetcher +from nf_core.pipelines.download.container_fetcher import ContainerFetcher from nf_core.pipelines.download.docker import DockerFetcher from nf_core.pipelines.download.singularity import SINGULARITY_CACHE_DIR_ENV_VAR, SingularityFetcher from nf_core.pipelines.download.utils import DownloadError @@ -250,7 +250,10 @@ def download_workflow(self) -> None: ] if self.container_system: summary_log.append(f"Container library: '{', '.join(self.container_library)}'") - if self.container_system in ("singularity", "apptainer") and os.environ.get(SINGULARITY_CACHE_DIR_ENV_VAR) is not None: + if ( + self.container_system in ("singularity", "apptainer") + and os.environ.get(SINGULARITY_CACHE_DIR_ENV_VAR) is not None + ): summary_log.append( f"Using [blue]{SINGULARITY_CACHE_DIR_ENV_VAR}[/]': {os.environ[SINGULARITY_CACHE_DIR_ENV_VAR]}'" ) From 6fac7caf8e25f4d7ad2682f5fe6084497bdf5152 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Sun, 16 Aug 2026 12:32:08 -0300 Subject: [PATCH 05/32] Add snapshot file for RunTestWorkflow (apple_container) --- .../snapshots/apple_container.nf.test.snap | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/snapshots/apple_container.nf.test.snap diff --git a/.github/snapshots/apple_container.nf.test.snap b/.github/snapshots/apple_container.nf.test.snap new file mode 100644 index 0000000000..5435135a0f --- /dev/null +++ b/.github/snapshots/apple_container.nf.test.snap @@ -0,0 +1,114 @@ +{ + "-profile test": { + "content": [ + { + "FASTQC": { + "fastqc": "0.12.1" + }, + "Workflow": { + "my-prefix/testpipeline": "v1.0.0dev" + } + }, + [ + "fastqc", + "fastqc/SAMPLE1_PE_1_fastqc.html", + "fastqc/SAMPLE1_PE_1_fastqc.zip", + "fastqc/SAMPLE1_PE_2_fastqc.html", + "fastqc/SAMPLE1_PE_2_fastqc.zip", + "fastqc/SAMPLE2_PE_1_fastqc.html", + "fastqc/SAMPLE2_PE_1_fastqc.zip", + "fastqc/SAMPLE2_PE_2_fastqc.html", + "fastqc/SAMPLE2_PE_2_fastqc.zip", + "fastqc/SAMPLE3_SE_1_fastqc.html", + "fastqc/SAMPLE3_SE_1_fastqc.zip", + "fastqc/SAMPLE3_SE_2_fastqc.html", + "fastqc/SAMPLE3_SE_2_fastqc.zip", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/fastqc-status-check-heatmap.txt", + "multiqc/multiqc_data/fastqc_overrepresented_sequences_plot.txt", + "multiqc/multiqc_data/fastqc_per_base_n_content_plot.txt", + "multiqc/multiqc_data/fastqc_per_base_sequence_quality_plot.txt", + "multiqc/multiqc_data/fastqc_per_sequence_gc_content_plot_Counts.txt", + "multiqc/multiqc_data/fastqc_per_sequence_gc_content_plot_Percentages.txt", + "multiqc/multiqc_data/fastqc_per_sequence_quality_scores_plot.txt", + "multiqc/multiqc_data/fastqc_sequence_counts_plot.txt", + "multiqc/multiqc_data/fastqc_sequence_duplication_levels_plot.txt", + "multiqc/multiqc_data/fastqc_sequence_length_distribution_plot.txt", + "multiqc/multiqc_data/fastqc_top_overrepresented_sequences_table.txt", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_fastqc.txt", + "multiqc/multiqc_data/multiqc_general_stats.txt", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/fastqc-status-check-heatmap.pdf", + "multiqc/multiqc_plots/pdf/fastqc_overrepresented_sequences_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_base_n_content_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_base_sequence_quality_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Counts.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Percentages.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_sequence_quality_scores_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_counts_plot-cnt.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_counts_plot-pct.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_duplication_levels_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_length_distribution_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_top_overrepresented_sequences_table.pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/fastqc-status-check-heatmap.png", + "multiqc/multiqc_plots/png/fastqc_overrepresented_sequences_plot.png", + "multiqc/multiqc_plots/png/fastqc_per_base_n_content_plot.png", + "multiqc/multiqc_plots/png/fastqc_per_base_sequence_quality_plot.png", + "multiqc/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Counts.png", + "multiqc/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Percentages.png", + "multiqc/multiqc_plots/png/fastqc_per_sequence_quality_scores_plot.png", + "multiqc/multiqc_plots/png/fastqc_sequence_counts_plot-cnt.png", + "multiqc/multiqc_plots/png/fastqc_sequence_counts_plot-pct.png", + "multiqc/multiqc_plots/png/fastqc_sequence_duplication_levels_plot.png", + "multiqc/multiqc_plots/png/fastqc_sequence_length_distribution_plot.png", + "multiqc/multiqc_plots/png/fastqc_top_overrepresented_sequences_table.png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/fastqc-status-check-heatmap.svg", + "multiqc/multiqc_plots/svg/fastqc_overrepresented_sequences_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_per_base_n_content_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_per_base_sequence_quality_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Counts.svg", + "multiqc/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Percentages.svg", + "multiqc/multiqc_plots/svg/fastqc_per_sequence_quality_scores_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_counts_plot-cnt.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_counts_plot-pct.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_duplication_levels_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_length_distribution_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_top_overrepresented_sequences_table.svg", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/testpipeline_software_mqc_versions.yml" + ], + [ + "fastqc-status-check-heatmap.txt:md5,0f1975c565a16bf09be08a05c204ded7", + "fastqc_overrepresented_sequences_plot.txt:md5,4b23cea39c4e23deef6b97810bc1ee46", + "fastqc_per_base_n_content_plot.txt:md5,037692101c0130c72493d3bbfa3afac1", + "fastqc_per_base_sequence_quality_plot.txt:md5,bfe735f3e31befe13bdf6761bb297d6e", + "fastqc_per_sequence_gc_content_plot_Counts.txt:md5,7108d19c46ef7883e864ba274c457d2e", + "fastqc_per_sequence_gc_content_plot_Percentages.txt:md5,23f527c80a148e4f34e5a43f6e520a90", + "fastqc_per_sequence_quality_scores_plot.txt:md5,a0cc0e6df7bfb05257da1cfc88b13c50", + "fastqc_sequence_counts_plot.txt:md5,c6e4e1588e6765fe8df27812a1322fbd", + "fastqc_sequence_duplication_levels_plot.txt:md5,3cde2db4033f6c64648976d1174db925", + "fastqc_sequence_length_distribution_plot.txt:md5,e82b9b14a7e24c0c5f27af97cebb6870", + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", + "multiqc_fastqc.txt:md5,1a41c2158adc9947bff9232962f70110", + "multiqc_general_stats.txt:md5,0b54e4e764665bd57fe0f95216744a78" + ] + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2025-06-16T14:29:10.076573" + } +} From 6a0f043584167389751da41639503c10545cacb2 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Wed, 19 Aug 2026 21:52:32 -0300 Subject: [PATCH 06/32] 'nf-core modules test module --profile' now supports apple container --- nf_core/__main__.py | 4 ++-- nf_core/components/components_test.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index c77eab1738..3ffcae7fa0 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1303,7 +1303,7 @@ def command_modules_create( ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda"]), + type=click.Choice(["docker", "singularity", "conda", "appleContainer"]), default=None, help="Run tests with a specific profile", ) @@ -1665,7 +1665,7 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force): ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda"]), + type=click.Choice(["docker", "singularity", "conda", "appleContainer"]), default=None, help="Run tests with a specific profile", ) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 587c724384..93d529eddb 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -145,10 +145,13 @@ def check_inputs(self) -> None: "type": "list", "name": "profile", "message": "Choose container software to run the test with", - "choices": ["Docker", "Singularity", "Conda"], + "choices": ["Docker", "Singularity", "Conda", "Apple Container"], } answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() + profile = answer["profile"] + # Map display names to Nextflow profile names + profile_map = {"Apple Container": "appleContainer"} + profile = profile_map.get(profile, profile.lower()) os.environ["PROFILE"] = profile def display_nftest_output(self, nftest_out: bytes, nftest_err: bytes) -> None: From 34cc95eb6cbb5455f4fcf5b0b42eccaa13a5e7b2 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Wed, 19 Aug 2026 22:14:38 -0300 Subject: [PATCH 07/32] Update Textual snapshots for Apple Container feature --- .../test_create_app/test_customisation_help.svg | 6 +++--- .../__snapshots__/test_create_app/test_type_custom.svg | 2 +- .../__snapshots__/test_create_app/test_type_nfcore.svg | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg index 90da8d5715..41adb4ba78 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg @@ -220,7 +220,7 @@ - + ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… @@ -237,8 +237,8 @@ ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ ▊▎Use a GitHub repository.Create a GitHub Show help  ▊▁▁▁▁▁▁▁▁▎repository for the▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -pipeline. -▇▇ +pipeline.▁▁ + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ ▊▎Add Github badgesThe README.md file of Hide help  ▊▁▁▁▁▁▁▁▁▎the pipeline will▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg index bb027a6d32..24e6fbe196 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg @@ -236,7 +236,7 @@ ▊▎Use a GitHub repository.Create a GitHub Show help  ▊▁▁▁▁▁▁▁▁▎repository for the▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ pipeline. -▂▂ +▃▃ ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ ▊▎Add Github badgesThe README.md file of Show help  ▊▁▁▁▁▁▁▁▁▎the pipeline will▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg index 2c6d37cfd5..1c2387d39d 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg @@ -214,7 +214,7 @@ - + ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… @@ -254,9 +254,9 @@ ▊▎Use GPUAdd GPU support to the Show help  ▊▁▁▁▁▁▁▁▁▎pipeline▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎Use Apple ContainerAdd Apple Container Show help  +▊▁▁▁▁▁▁▁▁▎support to the pipeline▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ From 57ab00f59b25d96973256c81927470a89dcba035 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Fri, 21 Aug 2026 00:32:31 -0300 Subject: [PATCH 08/32] feat(appleContainer): enable Wave arm64 provisioning and document limits Apple Container runs containers only natively on Apple Silicon (arm64) with no Rosetta/QEMU emulation, so it cannot run amd64 biocontainer images. Without an arm64 image source the appleContainer profile stalls trying to use incompatible images. Wire the appleContainer profile to build native arm64 images on demand via Wave from the module's conda packages: process.arch = 'arm64' wave.enabled = true wave.freeze = true wave.strategy = 'conda,container' Applied consistently to: - nf_core/pipeline-template/nextflow.config (generated pipelines) - tests/data/mock_pipeline_containers/nextflow.config (test fixture parity) - template_features.yml (documents behavior) Verified: nf-test on the seqkit/stats module with --profile appleContainer builds a native arm64 image via Wave and passes all tests. Document the limitations in template_features.yml: - Only tools with an arm64 build (conda linux-aarch64 package or multi-arch container image) can be provisioned. amd64-only tools such as prinseq-plus-plus and fastqc cannot run under Apple Container; use the docker profile with amd64 emulation for those. - The profile works without a Seqera token, but wave.freeze pushes anonymous frozen images to the community registry where they expire (causing occasional rebuilds). Setting TOWER_ACCESS_TOKEN is recommended so frozen images are durable and reproducible, after which runs simply pull the cached arm64 image. --- nf_core/pipeline-template/nextflow.config | 6 ++++++ nf_core/pipelines/create/template_features.yml | 18 ++++++++++++++++++ .../mock_pipeline_containers/nextflow.config | 6 ++++++ 3 files changed, 30 insertions(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 7ce397763e..7b1f18aa1d 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -208,6 +208,12 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + // Apple Container only runs arm64 natively on Apple Silicon, + // so use Wave to build arm64 images from conda packages + process.arch = 'arm64' + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } wave { apptainer.ociAutoPull = true diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 733ea326e6..8436278129 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -404,6 +404,24 @@ configurations: (`container`) with native `pull` and `save` commands. No Docker Desktop installation is required. + Since Apple Container does not emulate foreign architectures (no Rosetta/QEMU), + the profile automatically enables Wave with `process.arch = 'arm64'` to provision + native ARM64 container images from conda packages on demand. + + This only works for tools that have an ARM64 build available (a conda + `linux-aarch64` package or a multi-arch container image). Tools whose + upstream only ships amd64 builds (e.g. `prinseq-plus-plus`, `fastqc`) + cannot be provisioned this way — Apple Container will not run amd64 + images. For those tools, use the `docker` profile with amd64 emulation + (Docker Desktop / colima with Rosetta or QEMU) instead. + + The profile works without a Seqera token, but because it sets + `wave.freeze = true`, anonymous frozen images are pushed to the + community registry and expire over time (triggering occasional + rebuilds). Setting `TOWER_ACCESS_TOKEN` is recommended so frozen images + are durable and reproducible — after the first build, runs simply pull + the cached ARM64 image. + The `nf-core pipelines download` command will also support downloading images for offline use with `--container-system appleContainer`. nfcore_pipelines: True diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index 305edfb5c1..b68aee3224 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -158,6 +158,12 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + // Apple Container only runs arm64 natively on Apple Silicon, + // so use Wave to build arm64 images from conda packages + process.arch = 'arm64' + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } wave { apptainer.ociAutoPull = true From 617d9c77c228a21d7aade3eec6e63706e6b9043d Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Fri, 21 Aug 2026 00:50:28 -0300 Subject: [PATCH 09/32] Add mock build registry so CI stops complaining about nextflow inspect --- tests/data/mock_pipeline_containers/nextflow.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index b68aee3224..eefa87654e 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -164,6 +164,9 @@ profiles { wave.enabled = true wave.freeze = true wave.strategy = 'conda,container' + // Wave freeze mode requires a build repository to push images to. + // A dummy value is enough for `nextflow inspect` to build a valid request. + wave.build.repository = 'docker.io/example/mock-pipeline' } wave { apptainer.ociAutoPull = true From 3e8e9060facdbbf929d92109b0d8b4df1fb96694 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Fri, 21 Aug 2026 01:15:35 -0300 Subject: [PATCH 10/32] fix(test): don't route mock container fixture through Wave in appleContainer profile The test_containers_pipeline_apple_container test drives `nextflow inspect` under `-profile appleContainer,test,test_full` to verify container discovery. The fixture's appleContainer profile mirrored the production template and enabled Wave freeze (wave.enabled + wave.freeze + wave.strategy = 'conda,container', process.arch = 'arm64'). This made `nextflow inspect` call the live Wave API for every container in the mock pipeline, which cannot work in CI: - Freeze mode without a build repository is rejected outright: POST /v1alpha2/container [400] "Attribute `buildRepository` must be specified when using freeze mode". - Even with a build repository, the mock's `container` directives (e.g. quay.io/biocontainers/singlequay:1.9--pyh9f0ad1d_0) are pushed through Wave and fail on registry auth: POST /v1alpha2/container [401] "repository '...' unauthorized". Either way `nextflow inspect` dies on the Wave call before reaching the strict-syntax DownloadError the test asserts on, and the test depends on live Wave + registry auth, making it fragile. Container discovery via `nextflow inspect` does not need Wave: the fixture only needs the container system enabled so declared images resolve. This drops all wave.* settings from the fixture's appleContainer profile so it behaves like the sibling docker/apptainer/singularity profiles (none of which enable Wave). The production template profile (nf_core/pipeline-template/nextflow.config) is intentionally left unchanged; Wave freeze there is a runtime concern for real arm64 pipelines, not something this discovery test should exercise. --- .../data/mock_pipeline_containers/nextflow.config | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index eefa87654e..6ba9d46fe8 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -158,15 +158,12 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false - // Apple Container only runs arm64 natively on Apple Silicon, - // so use Wave to build arm64 images from conda packages - process.arch = 'arm64' - wave.enabled = true - wave.freeze = true - wave.strategy = 'conda,container' - // Wave freeze mode requires a build repository to push images to. - // A dummy value is enough for `nextflow inspect` to build a valid request. - wave.build.repository = 'docker.io/example/mock-pipeline' + // NOTE: The production template's appleContainer profile enables Wave + // freeze to build arm64 images from conda. That is deliberately omitted + // here: this fixture exercises container discovery via `nextflow inspect` + // and must not depend on the live Wave API / registry auth. Like the + // docker/apptainer/singularity profiles above, it only enables the + // container system so inspection can resolve the declared images. } wave { apptainer.ociAutoPull = true From 9ef288975f14328f9fee5e6c54c6da16ec7d88bc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:36:11 +0000 Subject: [PATCH 11/32] fix(ci): pin Nextflow version in CodSpeed workflow Co-authored-by: mribeirodantas <1023197+mribeirodantas@users.noreply.github.com> --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 0667bcd659..8d182cd4a4 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -39,7 +39,7 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914 # v3 with: - version: ${{ github.event.inputs.nextflow-version || matrix.nextflow-version }} + version: "25.10.4" - name: Install uv uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 From f4171b6fa49d4c5b18b0bfc8b124c65604081a20 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Sun, 23 Aug 2026 23:14:39 -0300 Subject: [PATCH 12/32] fix(appleContainer): emulate amd64 by default instead of Wave/arm64-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apple Container runs on Apple Silicon (arm64) but, like Docker, can run amd64 images via emulation with `--platform linux/amd64`. The previous support assumed Apple Container could only run native arm64 and forced every run through Wave to build arm64 images from conda — slow, network-dependent, and broken for tools without an arm64 build. Since most nf-core modules ship amd64 container images, the profile now emulates amd64 by default and works out of the box with no Wave dependency. Native arm64 via Wave becomes an explicit opt-in. - nextflow.config: appleContainer profile now sets `process.arch = 'linux/amd64'` and passes `--platform linux/amd64`; appends `--rosetta` when the new `--apple_rosetta` param is set. Add a composable `appleContainerWave` profile for native arm64 via Wave (`-profile appleContainer,appleContainerWave`). - template_features.yml: add the `apple_rosetta` param (default false, gated behind the apple_container feature) and rewrite the help text to document emulation, Rosetta, and the Wave opt-in. - containers_utils.py: add `appleContainer_amd64` alongside `appleContainer_arm64` so generated offline container configs match both profiles. - download/apple_container.py: make the pull/save `--platform` value configurable via `image_arch`, defaulting to linux/amd64 instead of a hardcoded linux/arm64. - Update mock fixture comment and download tests to expect linux/amd64. --- nf_core/pipeline-template/nextflow.config | 36 +++++++++++++--- nf_core/pipelines/containers_utils.py | 1 + .../pipelines/create/template_features.yml | 41 +++++++++++-------- nf_core/pipelines/download/apple_container.py | 10 ++++- .../mock_pipeline_containers/nextflow.config | 14 ++++--- .../download/test_apple_container.py | 4 +- 6 files changed, 73 insertions(+), 33 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 7b1f18aa1d..714bceffdd 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -51,6 +51,14 @@ params { {%- if test_config %} pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/'{% endif %} trace_report_suffix = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') + {%- if apple_container %} + + // Apple Container options + // When true, the appleContainer profile emulates amd64 images with Rosetta + // (faster) instead of the default plain amd64 emulation. Requires Rosetta + // installed in the Apple Container VM. + apple_rosetta = false + {%- endif %} {%- if nf_core_configs %} // Config options @@ -208,12 +216,28 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false - // Apple Container only runs arm64 natively on Apple Silicon, - // so use Wave to build arm64 images from conda packages - process.arch = 'arm64' - wave.enabled = true - wave.freeze = true - wave.strategy = 'conda,container' + // Apple Container runs natively on Apple Silicon (arm64), but like + // Docker it can also run amd64 images via emulation. Since most nf-core + // modules ship amd64 container images, emulate amd64 by default so + // standard pipelines run without any Wave dependency. Set + // `--apple_rosetta` to use faster Rosetta emulation (requires Rosetta + // installed in the Apple Container VM). + process.arch = 'linux/amd64' + appleContainer.runOptions = params.apple_rosetta ? '--platform linux/amd64 --rosetta' : '--platform linux/amd64' + } + appleContainerWave { + // Opt-in override: provision native arm64 images from conda via Wave + // instead of emulating amd64. Compose with the appleContainer profile, + // e.g. `-profile appleContainer,appleContainerWave`. Faster per task + // when an arm64 build exists, but only works for tools with an arm64 + // (conda linux-aarch64 / multi-arch) build available, and pushes frozen + // images to a registry (set TOWER_ACCESS_TOKEN for durable, + // reproducible builds). + process.arch = 'arm64' + appleContainer.runOptions = '' + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } wave { apptainer.ociAutoPull = true diff --git a/nf_core/pipelines/containers_utils.py b/nf_core/pipelines/containers_utils.py index 9a21fd7c1b..d0f88f8b39 100644 --- a/nf_core/pipelines/containers_utils.py +++ b/nf_core/pipelines/containers_utils.py @@ -17,6 +17,7 @@ "singularity_oras_arm64": ["singularity", "linux/arm64", "name"], "singularity_https_amd64": ["singularity", "linux/amd64", "https"], "singularity_https_arm64": ["singularity", "linux/arm64", "https"], + "appleContainer_amd64": ["docker", "linux/amd64", "name"], "appleContainer_arm64": ["docker", "linux/arm64", "name"], "conda_lock_files_amd64": ["conda", "linux/amd64", "lock_file"], "conda_lock_files_arm64": ["conda", "linux/arm64", "lock_file"], diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 8436278129..175f7dc683 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -404,23 +404,30 @@ configurations: (`container`) with native `pull` and `save` commands. No Docker Desktop installation is required. - Since Apple Container does not emulate foreign architectures (no Rosetta/QEMU), - the profile automatically enables Wave with `process.arch = 'arm64'` to provision - native ARM64 container images from conda packages on demand. - - This only works for tools that have an ARM64 build available (a conda - `linux-aarch64` package or a multi-arch container image). Tools whose - upstream only ships amd64 builds (e.g. `prinseq-plus-plus`, `fastqc`) - cannot be provisioned this way — Apple Container will not run amd64 - images. For those tools, use the `docker` profile with amd64 emulation - (Docker Desktop / colima with Rosetta or QEMU) instead. - - The profile works without a Seqera token, but because it sets - `wave.freeze = true`, anonymous frozen images are pushed to the - community registry and expire over time (triggering occasional - rebuilds). Setting `TOWER_ACCESS_TOKEN` is recommended so frozen images - are durable and reproducible — after the first build, runs simply pull - the cached ARM64 image. + Apple Container runs natively on Apple Silicon (arm64), but like Docker it + can also run amd64 images via emulation. Because most nf-core modules ship + amd64 container images, the `appleContainer` profile emulates amd64 by + default (`process.arch = 'linux/amd64'`, `--platform linux/amd64`), so + standard pipelines run out of the box with no Wave dependency and no + Seqera token. + + For faster emulation, set `--apple_rosetta` to run images through Rosetta + (`--rosetta`). This requires Rosetta to be installed in the Apple Container + VM. + + If you would rather run native ARM64 images built from conda packages via + Wave, compose the opt-in `appleContainerWave` profile alongside it: + + ```bash + nextflow run -profile appleContainer,appleContainerWave + ``` + + This is faster per task when a native arm64 build exists, but only works + for tools that have an ARM64 build available (a conda `linux-aarch64` + package or a multi-arch container image); tools whose upstream only ships + amd64 builds cannot be provisioned this way. It also sets `wave.freeze = + true`, so frozen images are pushed to a registry — set `TOWER_ACCESS_TOKEN` + for durable, reproducible builds. The `nf-core pipelines download` command will also support downloading images for offline use with `--container-system appleContainer`. diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py index b119ca0fb9..315a3d9585 100644 --- a/nf_core/pipelines/download/apple_container.py +++ b/nf_core/pipelines/download/apple_container.py @@ -47,6 +47,7 @@ def __init__( registry_set: Iterable[str], parallel: int = 4, hide_progress: bool = False, + image_arch: str = "linux/amd64", ): super().__init__( outdir=outdir, @@ -57,6 +58,11 @@ def __init__( ) # Override the container output directory name self._container_output_dir = outdir / "apple-container-images" + # Architecture to request when pulling/saving images. Apple Container runs + # amd64 images via emulation (matching the appleContainer profile default), + # so default to linux/amd64. Set to linux/arm64 for native arm64 images + # (parity with the opt-in appleContainerWave profile). + self.image_arch = image_arch def check_and_set_implementation(self) -> None: """ @@ -79,7 +85,7 @@ def construct_pull_command(self, address: str) -> list[str]: Args: address (str): The address of the container to pull. """ - pull_command = ["container", "image", "pull", "--platform", "linux/arm64", address] + pull_command = ["container", "image", "pull", "--platform", self.image_arch, address] log.debug(f"Apple Container command: {' '.join(pull_command)}") return pull_command @@ -96,7 +102,7 @@ def construct_save_command(self, output_path: Path, address: str) -> list[str]: "image", "save", "--platform", - "linux/arm64", + self.image_arch, "--output", str(output_path), address, diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index 6ba9d46fe8..1e5ddaef91 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -158,12 +158,14 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false - // NOTE: The production template's appleContainer profile enables Wave - // freeze to build arm64 images from conda. That is deliberately omitted - // here: this fixture exercises container discovery via `nextflow inspect` - // and must not depend on the live Wave API / registry auth. Like the - // docker/apptainer/singularity profiles above, it only enables the - // container system so inspection can resolve the declared images. + // The production template's appleContainer profile emulates amd64 images + // (process.arch = 'linux/amd64', --platform linux/amd64) and does NOT + // enable Wave by default. The runOptions/process.arch settings are + // omitted here because this fixture exercises container discovery via + // `nextflow inspect`: like the docker/apptainer/singularity profiles + // above, it only enables the container system so inspection can resolve + // the declared images. The opt-in appleContainerWave profile (Wave/arm64) + // is likewise omitted so inspection never depends on the live Wave API. } wave { apptainer.ociAutoPull = true diff --git a/tests/pipelines/download/test_apple_container.py b/tests/pipelines/download/test_apple_container.py index e554e637b2..1f024797bc 100644 --- a/tests/pipelines/download/test_apple_container.py +++ b/tests/pipelines/download/test_apple_container.py @@ -167,7 +167,7 @@ def test_apple_container_pull_command(self, tmp_path): registry_set=[], ) cmd = fetcher.construct_pull_command("quay.io/biocontainers/fastqc:0.12.1") - assert cmd == ["container", "image", "pull", "--platform", "linux/arm64", "quay.io/biocontainers/fastqc:0.12.1"] + assert cmd == ["container", "image", "pull", "--platform", "linux/amd64", "quay.io/biocontainers/fastqc:0.12.1"] # # Test that construct_save_command produces the correct command @@ -191,7 +191,7 @@ def test_apple_container_save_command(self, tmp_path): "image", "save", "--platform", - "linux/arm64", + "linux/amd64", "--output", str(output_path), "quay.io/biocontainers/fastqc:0.12.1", From e372c3827f4f921ef58dd49c981b588a9e3a8c98 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Mon, 24 Aug 2026 00:19:15 -0300 Subject: [PATCH 13/32] Fix linting error in GitHub Action --- nf_core/pipeline-template/nextflow_schema.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index a6caee0b3e..bdfd4ffee0 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -226,7 +226,14 @@ "fa_icon": "far calendar", "description": "Suffix to add to the trace report filename. Default is the date and time in the format yyyy-MM-dd_HH-mm-ss.", "hidden": true - }{% if nf_schema %}, + }{% if apple_container %}, + "apple_rosetta": { + "type": "boolean", + "fa_icon": "fab fa-apple", + "description": "Use faster Rosetta amd64 emulation in the appleContainer profile instead of plain amd64 emulation.", + "help_text": "Only applies to the `appleContainer` profile. When `true`, amd64 container images are emulated with Rosetta (faster) instead of the default plain amd64 emulation. Requires Rosetta to be installed in the Apple Container VM.", + "hidden": true + }{% endif %}{% if nf_schema %}, "help": { "type": ["boolean", "string"], "description": "Display the help message." From 867220dc4398994e9bfb36a958fc0dcc1ddd4939 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Mon, 24 Aug 2026 00:56:13 -0300 Subject: [PATCH 14/32] feat(appleContainer): make appleContainerWave a self-contained, selectable profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously appleContainerWave was an opt-in override meant only to be composed after appleContainer (`-profile appleContainer,appleContainerWave`). It relied on the base profile to disable the other container engines, so it could not be used on its own — which broke single-profile callers like `nf-core modules/subworkflows test --profile ...`. Make the profile stand alone by explicitly enabling appleContainer and disabling conda/docker/singularity/podman/shifter/charliecloud/apptainer, so it works both standalone (`-profile appleContainerWave`) and composed after appleContainer. Also expose it through the tooling: - Add "appleContainerWave" to the --profile choices for `modules create` and `subworkflows create`. - Add "Apple Container (Wave/arm64)" to the interactive test profile picker and map it to the appleContainerWave Nextflow profile. --- nf_core/__main__.py | 4 ++-- nf_core/components/components_test.py | 7 +++++-- nf_core/pipeline-template/nextflow.config | 25 ++++++++++++++++------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3ffcae7fa0..d75b18b728 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1303,7 +1303,7 @@ def command_modules_create( ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda", "appleContainer"]), + type=click.Choice(["docker", "singularity", "conda", "appleContainer", "appleContainerWave"]), default=None, help="Run tests with a specific profile", ) @@ -1665,7 +1665,7 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force): ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda", "appleContainer"]), + type=click.Choice(["docker", "singularity", "conda", "appleContainer", "appleContainerWave"]), default=None, help="Run tests with a specific profile", ) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 93d529eddb..43099cf7d7 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -145,12 +145,15 @@ def check_inputs(self) -> None: "type": "list", "name": "profile", "message": "Choose container software to run the test with", - "choices": ["Docker", "Singularity", "Conda", "Apple Container"], + "choices": ["Docker", "Singularity", "Conda", "Apple Container", "Apple Container (Wave/arm64)"], } answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"] # Map display names to Nextflow profile names - profile_map = {"Apple Container": "appleContainer"} + profile_map = { + "Apple Container": "appleContainer", + "Apple Container (Wave/arm64)": "appleContainerWave", + } profile = profile_map.get(profile, profile.lower()) os.environ["PROFILE"] = profile diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 714bceffdd..915e4cbb49 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -226,13 +226,24 @@ profiles { appleContainer.runOptions = params.apple_rosetta ? '--platform linux/amd64 --rosetta' : '--platform linux/amd64' } appleContainerWave { - // Opt-in override: provision native arm64 images from conda via Wave - // instead of emulating amd64. Compose with the appleContainer profile, - // e.g. `-profile appleContainer,appleContainerWave`. Faster per task - // when an arm64 build exists, but only works for tools with an arm64 - // (conda linux-aarch64 / multi-arch) build available, and pushes frozen - // images to a registry (set TOWER_ACCESS_TOKEN for durable, - // reproducible builds). + // Self-contained Apple Container + Wave profile: provision native arm64 + // images from conda via Wave instead of emulating amd64. Usable on its + // own (`-profile appleContainerWave`, e.g. with + // `nf-core modules test --profile appleContainer`-style single-profile + // callers) or composed after appleContainer + // (`-profile appleContainer,appleContainerWave`). Faster per task when + // an arm64 build exists, but only works for tools with an arm64 (conda + // linux-aarch64 / multi-arch) build available, and pushes frozen images + // to a registry (set TOWER_ACCESS_TOKEN for durable, reproducible + // builds). + appleContainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false process.arch = 'arm64' appleContainer.runOptions = '' wave.enabled = true From 21abd60f3d84ac3687186d2a535e5efde79b453c Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 25 Aug 2026 14:07:15 -0300 Subject: [PATCH 15/32] Apply suggestion from reviewer --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 915e4cbb49..06c922727e 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -56,7 +56,7 @@ params { // Apple Container options // When true, the appleContainer profile emulates amd64 images with Rosetta // (faster) instead of the default plain amd64 emulation. Requires Rosetta - // installed in the Apple Container VM. + // installed in the host OS. apple_rosetta = false {%- endif %} {%- if nf_core_configs %} From 81dea56bfe86536696c0939ba24c71e1fb50942a Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Mon, 31 Aug 2026 16:11:22 -0300 Subject: [PATCH 16/32] Apply batched suggestions from code review Co-authored-by: Pontus Freyhult --- nf_core/pipeline-template/nextflow_schema.json | 2 +- nf_core/pipelines/create/template_features.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index bdfd4ffee0..c0b74ddc08 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -231,7 +231,7 @@ "type": "boolean", "fa_icon": "fab fa-apple", "description": "Use faster Rosetta amd64 emulation in the appleContainer profile instead of plain amd64 emulation.", - "help_text": "Only applies to the `appleContainer` profile. When `true`, amd64 container images are emulated with Rosetta (faster) instead of the default plain amd64 emulation. Requires Rosetta to be installed in the Apple Container VM.", + "help_text": "Only applies to the `appleContainer` profile. When `true`, amd64 container images are emulated with Rosetta (faster) instead of the default plain amd64 emulation. Requires Rosetta to be installed on the host.", "hidden": true }{% endif %}{% if nf_schema %}, "help": { diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 175f7dc683..fd72c6709b 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -412,8 +412,7 @@ configurations: Seqera token. For faster emulation, set `--apple_rosetta` to run images through Rosetta - (`--rosetta`). This requires Rosetta to be installed in the Apple Container - VM. + (`--rosetta`). This requires Rosetta to be installed on the host. If you would rather run native ARM64 images built from conda packages via Wave, compose the opt-in `appleContainerWave` profile alongside it: From 55e2308482c53b1c58d1fc0493532ac2cb503c8d Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Mon, 31 Aug 2026 16:13:30 -0300 Subject: [PATCH 17/32] Revert this change --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 8d182cd4a4..0667bcd659 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -39,7 +39,7 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914 # v3 with: - version: "25.10.4" + version: ${{ github.event.inputs.nextflow-version || matrix.nextflow-version }} - name: Install uv uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 From c0b6844be1da14ac953ac6f74fd8784bddadc9b9 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Mon, 31 Aug 2026 16:16:49 -0300 Subject: [PATCH 18/32] add appleContainerWave to usage docs --- nf_core/pipeline-template/docs/usage.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 255a2cfc58..9398fcf9af 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -166,6 +166,10 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `appleContainer` - A generic configuration profile to be used with [Apple container](https://github.com/apple/container) on macOS (Apple silicon) + - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta emulation (requires Rosetta installed on the host. +- `appleContainerWave` + - An Apple container profile that provisions native arm64 images from conda via [Wave](https://seqera.io/wave/) instead of emulating amd64. Faster per task when an arm64 build exists, but only works for tools that have an arm64 build available (conda `linux-aarch64` or a multi-arch image). + - Can be used on its own (`-profile appleContainerWave`) or composed after `appleContainer` (`-profile appleContainer,appleContainerWave`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. - `wave` - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow `24.03.0-edge` or later). - `conda` From e8e10713b9f65daf55d5f69479557e44181cf6ce Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Mon, 31 Aug 2026 16:35:39 -0300 Subject: [PATCH 19/32] Update apple container profile names to snake case capitalization --- nf_core/__main__.py | 6 +++--- nf_core/components/components_test.py | 4 ++-- nf_core/pipeline-template/docs/usage.md | 6 +++--- nf_core/pipeline-template/nextflow.config | 14 +++++++------- nf_core/pipeline-template/nextflow_schema.json | 4 ++-- nf_core/pipelines/create/template_features.yml | 8 ++++---- nf_core/pipelines/download/apple_container.py | 4 ++-- nf_core/pipelines/download/download.py | 10 +++++----- .../data/mock_pipeline_containers/nextflow.config | 6 +++--- ...ainers.json => apple_container_containers.json} | 0 tests/pipelines/download/test_download.py | 4 ++-- 11 files changed, 33 insertions(+), 33 deletions(-) rename tests/data/mock_pipeline_containers/per_profile_output/{appleContainer_containers.json => apple_container_containers.json} (100%) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d75b18b728..3e9f88a098 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -342,7 +342,7 @@ def command_pipelines_lint( @click.option( "-s", "--container-system", - type=click.Choice(["none", "singularity", "docker", "apptainer", "appleContainer"]), + type=click.Choice(["none", "singularity", "docker", "apptainer", "apple_container"]), help="Download container images of required software.", ) @click.option( @@ -1303,7 +1303,7 @@ def command_modules_create( ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda", "appleContainer", "appleContainerWave"]), + type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_wave"]), default=None, help="Run tests with a specific profile", ) @@ -1665,7 +1665,7 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force): ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda", "appleContainer", "appleContainerWave"]), + type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_wave"]), default=None, help="Run tests with a specific profile", ) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 43099cf7d7..a0b9eddab4 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -151,8 +151,8 @@ def check_inputs(self) -> None: profile = answer["profile"] # Map display names to Nextflow profile names profile_map = { - "Apple Container": "appleContainer", - "Apple Container (Wave/arm64)": "appleContainerWave", + "Apple Container": "apple_container", + "Apple Container (Wave/arm64)": "apple_container_wave", } profile = profile_map.get(profile, profile.lower()) os.environ["PROFILE"] = profile diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 9398fcf9af..c34f4f56d2 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -164,12 +164,12 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://charliecloud.io/) - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) -- `appleContainer` +- `apple_container` - A generic configuration profile to be used with [Apple container](https://github.com/apple/container) on macOS (Apple silicon) - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta emulation (requires Rosetta installed on the host. -- `appleContainerWave` +- `apple_container_wave` - An Apple container profile that provisions native arm64 images from conda via [Wave](https://seqera.io/wave/) instead of emulating amd64. Faster per task when an arm64 build exists, but only works for tools that have an arm64 build available (conda `linux-aarch64` or a multi-arch image). - - Can be used on its own (`-profile appleContainerWave`) or composed after `appleContainer` (`-profile appleContainer,appleContainerWave`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. + - Can be used on its own (`-profile apple_container_wave`) or composed after `apple_container` (`-profile apple_container,apple_container_wave`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. - `wave` - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow `24.03.0-edge` or later). - `conda` diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 06c922727e..4111875c09 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -54,7 +54,7 @@ params { {%- if apple_container %} // Apple Container options - // When true, the appleContainer profile emulates amd64 images with Rosetta + // When true, the apple_container profile emulates amd64 images with Rosetta // (faster) instead of the default plain amd64 emulation. Requires Rosetta // installed in the host OS. apple_rosetta = false @@ -207,7 +207,7 @@ profiles { charliecloud.enabled = false appleContainer.enabled = false } - appleContainer { + apple_container { appleContainer.enabled = true conda.enabled = false docker.enabled = false @@ -225,13 +225,13 @@ profiles { process.arch = 'linux/amd64' appleContainer.runOptions = params.apple_rosetta ? '--platform linux/amd64 --rosetta' : '--platform linux/amd64' } - appleContainerWave { + apple_container_wave { // Self-contained Apple Container + Wave profile: provision native arm64 // images from conda via Wave instead of emulating amd64. Usable on its - // own (`-profile appleContainerWave`, e.g. with - // `nf-core modules test --profile appleContainer`-style single-profile - // callers) or composed after appleContainer - // (`-profile appleContainer,appleContainerWave`). Faster per task when + // own (`-profile apple_container_wave`, e.g. with + // `nf-core modules test --profile apple_container`-style single-profile + // callers) or composed after apple_container + // (`-profile apple_container,apple_container_wave`). Faster per task when // an arm64 build exists, but only works for tools with an arm64 (conda // linux-aarch64 / multi-arch) build available, and pushes frozen images // to a registry (set TOWER_ACCESS_TOKEN for durable, reproducible diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index c0b74ddc08..a1eb7f18af 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -230,8 +230,8 @@ "apple_rosetta": { "type": "boolean", "fa_icon": "fab fa-apple", - "description": "Use faster Rosetta amd64 emulation in the appleContainer profile instead of plain amd64 emulation.", - "help_text": "Only applies to the `appleContainer` profile. When `true`, amd64 container images are emulated with Rosetta (faster) instead of the default plain amd64 emulation. Requires Rosetta to be installed on the host.", + "description": "Use faster Rosetta amd64 emulation in the apple_container profile instead of plain amd64 emulation.", + "help_text": "Only applies to the `apple_container` profile. When `true`, amd64 container images are emulated with Rosetta (faster) instead of the default plain amd64 emulation. Requires Rosetta to be installed on the host.", "hidden": true }{% endif %}{% if nf_schema %}, "help": { diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index fd72c6709b..d51d81e1e3 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -396,7 +396,7 @@ configurations: short_description: "Use Apple Container" description: "Add Apple Container support to the pipeline" help_text: | - This will add an `appleContainer` profile to the pipeline configuration, enabling + This will add an `apple_container` profile to the pipeline configuration, enabling native container execution on macOS with Apple Silicon via the [Apple Container](https://github.com/apple/container) runtime. @@ -406,7 +406,7 @@ configurations: Apple Container runs natively on Apple Silicon (arm64), but like Docker it can also run amd64 images via emulation. Because most nf-core modules ship - amd64 container images, the `appleContainer` profile emulates amd64 by + amd64 container images, the `apple_container` profile emulates amd64 by default (`process.arch = 'linux/amd64'`, `--platform linux/amd64`), so standard pipelines run out of the box with no Wave dependency and no Seqera token. @@ -415,10 +415,10 @@ configurations: (`--rosetta`). This requires Rosetta to be installed on the host. If you would rather run native ARM64 images built from conda packages via - Wave, compose the opt-in `appleContainerWave` profile alongside it: + Wave, compose the opt-in `apple_container_wave` profile alongside it: ```bash - nextflow run -profile appleContainer,appleContainerWave + nextflow run -profile apple_container,apple_container_wave ``` This is faster per task when a native arm64 build exists, but only works diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py index 315a3d9585..26500f09ea 100644 --- a/nf_core/pipelines/download/apple_container.py +++ b/nf_core/pipelines/download/apple_container.py @@ -59,9 +59,9 @@ def __init__( # Override the container output directory name self._container_output_dir = outdir / "apple-container-images" # Architecture to request when pulling/saving images. Apple Container runs - # amd64 images via emulation (matching the appleContainer profile default), + # amd64 images via emulation (matching the apple_container profile default), # so default to linux/amd64. Set to linux/arm64 for native arm64 images - # (parity with the opt-in appleContainerWave profile). + # (parity with the opt-in apple_container_wave profile). self.image_arch = image_arch def check_and_set_implementation(self) -> None: diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index 5ee0e99e88..fbcd3386a7 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -88,7 +88,7 @@ def __init__( ): # Verify that the flags provided make sense together if ( - container_system in ("docker", "appleContainer") + container_system in ("docker", "apple_container") and container_cache_utilisation != "copy" and container_cache_utilisation is not None ): @@ -334,7 +334,7 @@ def download_workflow_static(self) -> None: raise DownloadError("Error editing pipeline config file to use local configs!") from e # Collect all required container images - if self.container_system in ("singularity", "apptainer", "docker", "appleContainer"): + if self.container_system in ("singularity", "apptainer", "docker", "apple_container"): workflow_directory = self.outdir / revision_dirname self.find_container_images(workflow_directory, revision) @@ -370,7 +370,7 @@ def download_workflow_platform(self, location: Path | None = None) -> None: self.workflow_repo.bare_clone(self.output_filename) # extract the required containers - if self.container_system in ("singularity", "apptainer", "docker", "appleContainer"): + if self.container_system in ("singularity", "apptainer", "docker", "apple_container"): for revision, commit in self.wf_sha.items(): # Checkout the repo in the current revision self.workflow_repo.checkout(commit) @@ -506,7 +506,7 @@ def prompt_container_download(self) -> None: stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container_system = questionary.select( "Download software container images:", - choices=["none", "singularity", "docker", "apptainer", "appleContainer"], + choices=["none", "singularity", "docker", "apptainer", "apple_container"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -527,7 +527,7 @@ def setup_container_fetcher(self) -> None: hide_progress=self.hide_progress, container_system=self.container_system, ) - elif self.container_system == "appleContainer": + elif self.container_system == "apple_container": self.container_fetcher = AppleContainerFetcher( outdir=self.outdir, registry_set=self.registry_set, diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index 1e5ddaef91..9c881b2e07 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -149,7 +149,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false } - appleContainer { + apple_container { appleContainer.enabled = true conda.enabled = false docker.enabled = false @@ -158,13 +158,13 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false - // The production template's appleContainer profile emulates amd64 images + // The production template's apple_container profile emulates amd64 images // (process.arch = 'linux/amd64', --platform linux/amd64) and does NOT // enable Wave by default. The runOptions/process.arch settings are // omitted here because this fixture exercises container discovery via // `nextflow inspect`: like the docker/apptainer/singularity profiles // above, it only enables the container system so inspection can resolve - // the declared images. The opt-in appleContainerWave profile (Wave/arm64) + // the declared images. The opt-in apple_container_wave profile (Wave/arm64) // is likewise omitted so inspection never depends on the live Wave API. } wave { diff --git a/tests/data/mock_pipeline_containers/per_profile_output/appleContainer_containers.json b/tests/data/mock_pipeline_containers/per_profile_output/apple_container_containers.json similarity index 100% rename from tests/data/mock_pipeline_containers/per_profile_output/appleContainer_containers.json rename to tests/data/mock_pipeline_containers/per_profile_output/apple_container_containers.json diff --git a/tests/pipelines/download/test_download.py b/tests/pipelines/download/test_download.py index d1b4639db7..e91ccc5132 100644 --- a/tests/pipelines/download/test_download.py +++ b/tests/pipelines/download/test_download.py @@ -367,10 +367,10 @@ def test_containers_pipeline_apple_container(self, tmp_path, mock_fetch_wf_confi assert check_nextflow_version(NF_INSPECT_MIN_NF_VERSION) is True # Set up test - container_system = "appleContainer" + container_system = "apple_container" mock_pipeline_dir = TEST_DATA_DIR / "mock_pipeline_containers" refererence_json_dir = mock_pipeline_dir / "per_profile_output" - # Check that `-profile appleContainer` produces the same output as the reference + # Check that `-profile apple_container` produces the same output as the reference download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path, container_system=container_system) mock_fetch_wf_config.return_value = {} From 3ed849478fd64de19ed0c0d09d0ae22dffd0767a Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Fri, 9 Oct 2026 17:43:31 -0300 Subject: [PATCH 20/32] Rename the apple_container_wave profile to apple_container_arm64 --- nf_core/__main__.py | 4 ++-- nf_core/components/components_test.py | 4 ++-- nf_core/pipeline-template/docs/usage.md | 4 ++-- nf_core/pipeline-template/nextflow.config | 8 ++++---- nf_core/pipelines/create/template_features.yml | 4 ++-- nf_core/pipelines/download/apple_container.py | 2 +- tests/data/mock_pipeline_containers/nextflow.config | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3e9f88a098..38b9653c8f 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1303,7 +1303,7 @@ def command_modules_create( ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_wave"]), + type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_arm64"]), default=None, help="Run tests with a specific profile", ) @@ -1665,7 +1665,7 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force): ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_wave"]), + type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_arm64"]), default=None, help="Run tests with a specific profile", ) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index a0b9eddab4..7c5dcd5756 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -145,14 +145,14 @@ def check_inputs(self) -> None: "type": "list", "name": "profile", "message": "Choose container software to run the test with", - "choices": ["Docker", "Singularity", "Conda", "Apple Container", "Apple Container (Wave/arm64)"], + "choices": ["Docker", "Singularity", "Conda", "Apple Container", "Apple Container (arm64)"], } answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"] # Map display names to Nextflow profile names profile_map = { "Apple Container": "apple_container", - "Apple Container (Wave/arm64)": "apple_container_wave", + "Apple Container (arm64)": "apple_container_arm64", } profile = profile_map.get(profile, profile.lower()) os.environ["PROFILE"] = profile diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index c34f4f56d2..da5993f3d4 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -167,9 +167,9 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - `apple_container` - A generic configuration profile to be used with [Apple container](https://github.com/apple/container) on macOS (Apple silicon) - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta emulation (requires Rosetta installed on the host. -- `apple_container_wave` +- `apple_container_arm64` - An Apple container profile that provisions native arm64 images from conda via [Wave](https://seqera.io/wave/) instead of emulating amd64. Faster per task when an arm64 build exists, but only works for tools that have an arm64 build available (conda `linux-aarch64` or a multi-arch image). - - Can be used on its own (`-profile apple_container_wave`) or composed after `apple_container` (`-profile apple_container,apple_container_wave`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. + - Can be used on its own (`-profile apple_container_arm64`) or composed after `apple_container` (`-profile apple_container,apple_container_arm64`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. - `wave` - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow `24.03.0-edge` or later). - `conda` diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 4111875c09..4f1163a928 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -221,17 +221,17 @@ profiles { // modules ship amd64 container images, emulate amd64 by default so // standard pipelines run without any Wave dependency. Set // `--apple_rosetta` to use faster Rosetta emulation (requires Rosetta - // installed in the Apple Container VM). + // installed on the host). process.arch = 'linux/amd64' appleContainer.runOptions = params.apple_rosetta ? '--platform linux/amd64 --rosetta' : '--platform linux/amd64' } - apple_container_wave { + apple_container_arm64 { // Self-contained Apple Container + Wave profile: provision native arm64 // images from conda via Wave instead of emulating amd64. Usable on its - // own (`-profile apple_container_wave`, e.g. with + // own (`-profile apple_container_arm64`, e.g. with // `nf-core modules test --profile apple_container`-style single-profile // callers) or composed after apple_container - // (`-profile apple_container,apple_container_wave`). Faster per task when + // (`-profile apple_container,apple_container_arm64`). Faster per task when // an arm64 build exists, but only works for tools with an arm64 (conda // linux-aarch64 / multi-arch) build available, and pushes frozen images // to a registry (set TOWER_ACCESS_TOKEN for durable, reproducible diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index d51d81e1e3..2e69982345 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -415,10 +415,10 @@ configurations: (`--rosetta`). This requires Rosetta to be installed on the host. If you would rather run native ARM64 images built from conda packages via - Wave, compose the opt-in `apple_container_wave` profile alongside it: + Wave, compose the opt-in `apple_container_arm64` profile alongside it: ```bash - nextflow run -profile apple_container,apple_container_wave + nextflow run -profile apple_container,apple_container_arm64 ``` This is faster per task when a native arm64 build exists, but only works diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py index 26500f09ea..909207d0e9 100644 --- a/nf_core/pipelines/download/apple_container.py +++ b/nf_core/pipelines/download/apple_container.py @@ -61,7 +61,7 @@ def __init__( # Architecture to request when pulling/saving images. Apple Container runs # amd64 images via emulation (matching the apple_container profile default), # so default to linux/amd64. Set to linux/arm64 for native arm64 images - # (parity with the opt-in apple_container_wave profile). + # (parity with the opt-in apple_container_arm64 profile). self.image_arch = image_arch def check_and_set_implementation(self) -> None: diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index 9c881b2e07..5d99338301 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -164,7 +164,7 @@ profiles { // omitted here because this fixture exercises container discovery via // `nextflow inspect`: like the docker/apptainer/singularity profiles // above, it only enables the container system so inspection can resolve - // the declared images. The opt-in apple_container_wave profile (Wave/arm64) + // the declared images. The opt-in apple_container_arm64 profile (native arm64 via Wave) // is likewise omitted so inspection never depends on the live Wave API. } wave { From 18239ea2489e374bd809bfd14067c6da96ec2c98 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Fri, 9 Oct 2026 17:49:47 -0300 Subject: [PATCH 21/32] Accept reviewer comments by @mashehu --- nf_core/pipeline-template/nextflow.config | 13 +++--------- .../pipelines/create/template_features.yml | 20 +------------------ 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 4f1163a928..b9e0252aea 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -226,16 +226,9 @@ profiles { appleContainer.runOptions = params.apple_rosetta ? '--platform linux/amd64 --rosetta' : '--platform linux/amd64' } apple_container_arm64 { - // Self-contained Apple Container + Wave profile: provision native arm64 - // images from conda via Wave instead of emulating amd64. Usable on its - // own (`-profile apple_container_arm64`, e.g. with - // `nf-core modules test --profile apple_container`-style single-profile - // callers) or composed after apple_container - // (`-profile apple_container,apple_container_arm64`). Faster per task when - // an arm64 build exists, but only works for tools with an arm64 (conda - // linux-aarch64 / multi-arch) build available, and pushes frozen images - // to a registry (set TOWER_ACCESS_TOKEN for durable, reproducible - // builds). + // Native arm64 images built from conda via Wave instead of amd64 emulation. + // Only works for tools with an arm64 build. Set TOWER_ACCESS_TOKEN for + // durable, reproducible builds. appleContainer.enabled = true conda.enabled = false docker.enabled = false diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 2e69982345..5740353d12 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -400,10 +400,6 @@ configurations: native container execution on macOS with Apple Silicon via the [Apple Container](https://github.com/apple/container) runtime. - Apple Container uses standard OCI/Docker images and provides its own CLI - (`container`) with native `pull` and `save` commands. No Docker Desktop - installation is required. - Apple Container runs natively on Apple Silicon (arm64), but like Docker it can also run amd64 images via emulation. Because most nf-core modules ship amd64 container images, the `apple_container` profile emulates amd64 by @@ -412,21 +408,7 @@ configurations: Seqera token. For faster emulation, set `--apple_rosetta` to run images through Rosetta - (`--rosetta`). This requires Rosetta to be installed on the host. - - If you would rather run native ARM64 images built from conda packages via - Wave, compose the opt-in `apple_container_arm64` profile alongside it: - - ```bash - nextflow run -profile apple_container,apple_container_arm64 - ``` - - This is faster per task when a native arm64 build exists, but only works - for tools that have an ARM64 build available (a conda `linux-aarch64` - package or a multi-arch container image); tools whose upstream only ships - amd64 builds cannot be provisioned this way. It also sets `wave.freeze = - true`, so frozen images are pushed to a registry — set `TOWER_ACCESS_TOKEN` - for durable, reproducible builds. + . This requires Rosetta to be installed on the host. The `nf-core pipelines download` command will also support downloading images for offline use with `--container-system appleContainer`. From 8c744fc4b88b4550d40acebab0903364eb873686 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Sat, 10 Oct 2026 02:00:26 -0300 Subject: [PATCH 22/32] Make comment less verbose --- nf_core/pipeline-template/nextflow.config | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index e3210b0c3c..efedade8fd 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -216,12 +216,8 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false - // Apple Container runs natively on Apple Silicon (arm64), but like - // Docker it can also run amd64 images via emulation. Since most nf-core - // modules ship amd64 container images, emulate amd64 by default so - // standard pipelines run without any Wave dependency. Set - // `--apple_rosetta` to use faster Rosetta emulation (requires Rosetta - // installed on the host). + // Default to amd64 emulation for nf-core compatibility. + // Use --apple_rosetta for faster emulation. process.arch = 'linux/amd64' appleContainer.runOptions = params.apple_rosetta ? '--platform linux/amd64 --rosetta' : '--platform linux/amd64' } From 722a9eb8a6f315abae32212e3a0a4447f68d59e3 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 12:42:10 +0200 Subject: [PATCH 23/32] Update nf_core/pipeline-template/docs/usage.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index da5993f3d4..caecf955a1 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -166,7 +166,7 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `apple_container` - A generic configuration profile to be used with [Apple container](https://github.com/apple/container) on macOS (Apple silicon) - - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta emulation (requires Rosetta installed on the host. + - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta emulation (requires Rosetta 2 installed on the host. - `apple_container_arm64` - An Apple container profile that provisions native arm64 images from conda via [Wave](https://seqera.io/wave/) instead of emulating amd64. Faster per task when an arm64 build exists, but only works for tools that have an arm64 build available (conda `linux-aarch64` or a multi-arch image). - Can be used on its own (`-profile apple_container_arm64`) or composed after `apple_container` (`-profile apple_container,apple_container_arm64`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. From 16edf251d2af3337a846768e07fab03ea333406e Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 12:45:06 +0200 Subject: [PATCH 24/32] Apply batched suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/nextflow.config | 8 ++------ nf_core/pipelines/create/template_features.yml | 10 +++------- nf_core/pipelines/download/apple_container.py | 14 -------------- .../data/mock_pipeline_containers/nextflow.config | 8 -------- 4 files changed, 5 insertions(+), 35 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index efedade8fd..fa1990941a 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -54,9 +54,6 @@ params { {%- if apple_container %} // Apple Container options - // When true, the apple_container profile emulates amd64 images with Rosetta - // (faster) instead of the default plain amd64 emulation. Requires Rosetta - // installed in the host OS. apple_rosetta = false {%- endif %} {%- if nf_core_configs %} @@ -223,8 +220,7 @@ profiles { } apple_container_arm64 { // Native arm64 images built from conda via Wave instead of amd64 emulation. - // Only works for tools with an arm64 build. Set TOWER_ACCESS_TOKEN for - // durable, reproducible builds. + // Only works for tools with an arm64 build. appleContainer.enabled = true conda.enabled = false docker.enabled = false @@ -289,10 +285,10 @@ includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !pa // Set to your registry if you have a mirror of containers apptainer.registry = 'quay.io' appleContainer.registry = 'quay.io' +charliecloud.registry = 'quay.io' docker.registry = 'quay.io' podman.registry = 'quay.io' singularity.registry = 'quay.io' -charliecloud.registry = 'quay.io' {% if igenomes -%} // Load igenomes.config if required diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 5740353d12..b7bc338bd6 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -404,14 +404,10 @@ configurations: can also run amd64 images via emulation. Because most nf-core modules ship amd64 container images, the `apple_container` profile emulates amd64 by default (`process.arch = 'linux/amd64'`, `--platform linux/amd64`), so - standard pipelines run out of the box with no Wave dependency and no - Seqera token. + standard pipelines run out of the box. - For faster emulation, set `--apple_rosetta` to run images through Rosetta - . This requires Rosetta to be installed on the host. - - The `nf-core pipelines download` command will also support downloading - images for offline use with `--container-system appleContainer`. + For faster emulation, set `--apple_rosetta` to run images through Rosetta. + This requires Rosetta to be installed on the host. nfcore_pipelines: True custom_pipelines: True default: true diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py index 909207d0e9..fd7d7e689c 100644 --- a/nf_core/pipelines/download/apple_container.py +++ b/nf_core/pipelines/download/apple_container.py @@ -1,13 +1,5 @@ """Apple Container fetcher for nf-core download. -Apple Container (https://github.com/apple/container) is a lightweight -container runtime for macOS on Apple Silicon. It uses standard OCI/Docker -images and provides its own CLI (``container``) with native ``pull`` and -``save`` commands for offline bundling. - -Images are pulled and saved using the ``container`` CLI directly — no -Docker installation is required. Loading images on the target machine -uses ``container image load``. """ import logging @@ -35,8 +27,6 @@ class AppleContainerFetcher(DockerFetcher): """ Fetcher for Apple Container images. - Subclasses :class:`DockerFetcher` — uses the native Apple Container CLI - (``container image pull`` / ``container image save``) instead of Docker. The cleanup step writes an Apple-Container-specific load script. """ @@ -58,10 +48,6 @@ def __init__( ) # Override the container output directory name self._container_output_dir = outdir / "apple-container-images" - # Architecture to request when pulling/saving images. Apple Container runs - # amd64 images via emulation (matching the apple_container profile default), - # so default to linux/amd64. Set to linux/arm64 for native arm64 images - # (parity with the opt-in apple_container_arm64 profile). self.image_arch = image_arch def check_and_set_implementation(self) -> None: diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index 5d99338301..cc80af1daf 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -158,14 +158,6 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false - // The production template's apple_container profile emulates amd64 images - // (process.arch = 'linux/amd64', --platform linux/amd64) and does NOT - // enable Wave by default. The runOptions/process.arch settings are - // omitted here because this fixture exercises container discovery via - // `nextflow inspect`: like the docker/apptainer/singularity profiles - // above, it only enables the container system so inspection can resolve - // the declared images. The opt-in apple_container_arm64 profile (native arm64 via Wave) - // is likewise omitted so inspection never depends on the live Wave API. } wave { apptainer.ociAutoPull = true From 373b26a523cca7ce271175f5b4be61bb3883a72c Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 12:47:23 +0200 Subject: [PATCH 25/32] Apply batched suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipelines/download/apple_container.py | 2 +- nf_core/pipelines/download/docker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py index fd7d7e689c..98206e5e66 100644 --- a/nf_core/pipelines/download/apple_container.py +++ b/nf_core/pipelines/download/apple_container.py @@ -62,7 +62,7 @@ def check_and_set_implementation(self) -> None: "See: https://github.com/apple/container" ) - self.implementation = "container" + self.implementation = "apple container" def construct_pull_command(self, address: str) -> list[str]: """ diff --git a/nf_core/pipelines/download/docker.py b/nf_core/pipelines/download/docker.py index 6c3bce6e40..247a1579ba 100644 --- a/nf_core/pipelines/download/docker.py +++ b/nf_core/pipelines/download/docker.py @@ -100,7 +100,7 @@ def gather_registries(self, workflow_directory: Path) -> set[str]: set[str]: The set of registries to use for the container download """ registry_set = self.base_registry_set.copy() - configured_registry_keys = ["appleContainer.registry", "docker.registry", "podman.registry"] + configured_registry_keys = ["docker.registry", "podman.registry"] # Add the registries defined in the workflow config registry_set |= self.gather_config_registries( From 6b88ee42a935613128c5bfe42ab82fcc4bc783bd Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 12:48:17 +0200 Subject: [PATCH 26/32] Apply suggestion from @mashehu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipelines/download/singularity.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipelines/download/singularity.py b/nf_core/pipelines/download/singularity.py index 3f57912953..98d4b5981a 100644 --- a/nf_core/pipelines/download/singularity.py +++ b/nf_core/pipelines/download/singularity.py @@ -228,7 +228,6 @@ def gather_registries(self, workflow_directory: Path) -> set[str]: # Select registries defined in pipeline config configured_registry_keys = [ - "appleContainer.registry", "apptainer.registry", "docker.registry", "podman.registry", From f87bc4075884cd39343129d71e45cf507bfd4b4c Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 13:15:46 +0200 Subject: [PATCH 27/32] Check that Apple Container services are running before download AppleContainerFetcher only checked that the `container` CLI was on PATH. It now also runs `container system status`, the same way DockerFetcher runs `docker info`, and raises an OSError telling the user to run `container system start` if the services are stopped. Tests now mock the status check, so they don't need the services running on the test machine. The stale `implementation == "container"` assertion now expects "apple container", and a new test covers the services-not-running case. --- nf_core/pipelines/download/apple_container.py | 14 +++++++---- .../download/test_apple_container.py | 24 ++++++++++++++++++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py index 98206e5e66..aac996575a 100644 --- a/nf_core/pipelines/download/apple_container.py +++ b/nf_core/pipelines/download/apple_container.py @@ -1,6 +1,4 @@ -"""Apple Container fetcher for nf-core download. - -""" +"""Apple Container fetcher for nf-core download.""" import logging import shutil @@ -52,7 +50,7 @@ def __init__( def check_and_set_implementation(self) -> None: """ - Check if Apple Container CLI is installed and set the implementation. + Check that the Apple Container CLI is installed and its system services are running. """ container_binary = shutil.which("container") if not container_binary: @@ -62,6 +60,14 @@ def check_and_set_implementation(self) -> None: "See: https://github.com/apple/container" ) + try: + nf_core.utils.run_cmd(container_binary, "system status") + except RuntimeError as e: + raise OSError( + "Apple Container system services are required to pull images, but they are not running.\n" + "Start them with: container system start" + ) from e + self.implementation = "apple container" def construct_pull_command(self, address: str) -> list[str]: diff --git a/tests/pipelines/download/test_apple_container.py b/tests/pipelines/download/test_apple_container.py index 1f024797bc..e8c2fb76e1 100644 --- a/tests/pipelines/download/test_apple_container.py +++ b/tests/pipelines/download/test_apple_container.py @@ -20,6 +20,12 @@ class AppleContainerTest(unittest.TestCase): def use_caplog(self, caplog): self._caplog = caplog + def setUp(self): + # Don't depend on the Apple Container system services being started on the test host + patcher = mock.patch("nf_core.utils.run_cmd", return_value=(b"", b"")) + self.mock_run_cmd = patcher.start() + self.addCleanup(patcher.stop) + # # Test for AppleContainerFetcher initialization and output directory # @@ -135,7 +141,23 @@ def test_apple_container_check_implementation(self, tmp_path): container_library=[], registry_set=[], ) - assert fetcher.implementation == "container" + assert fetcher.implementation == "apple container" + self.mock_run_cmd.assert_called_once_with(shutil.which("container"), "system status") + + # + # Test that check_and_set_implementation raises when the system services are not running + # + @with_temporary_folder + @mock.patch("shutil.which", return_value="/usr/local/bin/container") + def test_apple_container_services_not_running_raises(self, tmp_path, mock_which): + self.mock_run_cmd.side_effect = RuntimeError("apiserver is not running") + with pytest.raises(OSError, match="container system start"): + AppleContainerFetcher( + outdir=Path(tmp_path), + container_library=[], + registry_set=[], + ) + self.mock_run_cmd.assert_called_once_with("/usr/local/bin/container", "system status") # # Test that check_and_set_implementation raises when container CLI is missing From 17561b0994ceb964c52d3b6962110c6ab72e06b7 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 13:40:09 +0200 Subject: [PATCH 28/32] Detect Apple Container pull/save failures and add integration tests DockerFetcher only raised on Docker daemon messages ("invalid reference format", "Error response from daemon:"). The Apple `container` CLI reports failures as "Error: ..." lines instead, so failed pulls and saves were silently treated as successes. - Move the error substrings into an overridable `error_markers` class attribute; AppleContainerFetcher sets it to ("Error:",) - Map Apple's "failed to get image for reference" to ImageNotPulledError - Add integration tests for pull_image, save_image, pull_and_save_image and fetch_containers that run the real `container` CLI; they are skipped when it is not installed or its services are not running The tests record a known `container` CLI v1.2.2 limitation: Seqera Containers images (bare single-platform OCI manifests) are rejected with "does not list any supported platform". --- nf_core/pipelines/download/apple_container.py | 3 + nf_core/pipelines/download/docker.py | 29 ++-- .../download/test_apple_container.py | 134 ++++++++++++++++++ 3 files changed, 151 insertions(+), 15 deletions(-) diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py index aac996575a..cd8af60d47 100644 --- a/nf_core/pipelines/download/apple_container.py +++ b/nf_core/pipelines/download/apple_container.py @@ -28,6 +28,9 @@ class AppleContainerFetcher(DockerFetcher): The cleanup step writes an Apple-Container-specific load script. """ + # The `container` CLI reports failures as `Error: ...` lines, not with Docker's daemon messages + error_markers: tuple[str, ...] = ("Error:",) + def __init__( self, outdir: Path, diff --git a/nf_core/pipelines/download/docker.py b/nf_core/pipelines/download/docker.py index 247a1579ba..0e9ff23c30 100644 --- a/nf_core/pipelines/download/docker.py +++ b/nf_core/pipelines/download/docker.py @@ -44,6 +44,9 @@ class DockerFetcher(ContainerFetcher): Fetcher for Docker containers. """ + # Substrings in the CLI output that mark a failed pull/save command + error_markers: tuple[str, ...] = ("invalid reference format", "Error response from daemon:") + def __init__( self, outdir: Path, @@ -313,21 +316,16 @@ def _run_docker_command( proc.wait() log.debug(f"Docker command '{' '.join(command)}' has exited.") - if lines: - # something went wrong with the container retrieval - possible_error_lines = { - "invalid reference format", - "Error response from daemon:", - } - if any(pel in line for pel in possible_error_lines for line in lines): - self.progress.remove_task(progress_task) - raise DockerError( - container=container, - address=address, - out_path=output_path, - command=command, - error_msg=lines, - ) + # something went wrong with the container retrieval + if lines and any(marker in line for marker in self.error_markers for line in lines): + self.progress.remove_task(progress_task) + raise DockerError( + container=container, + address=address, + out_path=output_path, + command=command, + error_msg=lines, + ) def cleanup(self) -> None: """ @@ -382,6 +380,7 @@ def __init__( error_patterns = { r"reference does not exist": self.ImageNotPulledError, + r"failed to get image for reference": self.ImageNotPulledError, # Apple Container r"repository does not exist": self.ImageNotFoundError, r"Error response from daemon: Head .*: denied": self.ImageNotFoundError, r"manifest unknown": self.InvalidTagError, diff --git a/tests/pipelines/download/test_apple_container.py b/tests/pipelines/download/test_apple_container.py index e8c2fb76e1..f0d9e658ea 100644 --- a/tests/pipelines/download/test_apple_container.py +++ b/tests/pipelines/download/test_apple_container.py @@ -2,6 +2,8 @@ import os import shutil +import subprocess +import tarfile import unittest from contextlib import redirect_stderr from io import StringIO @@ -11,10 +13,25 @@ import pytest from nf_core.pipelines.download.apple_container import AppleContainerFetcher +from nf_core.pipelines.download.docker import DockerError from ...utils import with_temporary_folder +def _apple_container_running() -> bool: + """True if the `container` CLI is installed and its system services are started.""" + container_binary = shutil.which("container") + if container_binary is None: + return False + return subprocess.run([container_binary, "system", "status"], capture_output=True, check=False).returncode == 0 + + +requires_apple_container = pytest.mark.skipif( + not _apple_container_running(), + reason="Can't test what Apple Container does if it's not installed and running.", +) + + class AppleContainerTest(unittest.TestCase): @pytest.fixture(autouse=True) def use_caplog(self, caplog): @@ -218,3 +235,120 @@ def test_apple_container_save_command(self, tmp_path): str(output_path), "quay.io/biocontainers/fastqc:0.12.1", ] + + # + # Integration tests: these run the real `container` CLI against public registries + # + def _make_fetcher(self, tmp_path: Path) -> AppleContainerFetcher: + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + fetcher.progress = mock.MagicMock() + return fetcher + + # + # Tests for 'pull_image' + # + @requires_apple_container + @with_temporary_folder + def test_apple_container_pull_image(self, tmp_path): + fetcher = self._make_fetcher(Path(tmp_path)) + task = mock.MagicMock() + + # Successful pulls: short name and absolute URI + fetcher.pull_image("hello-world", task) + fetcher.pull_image("docker.io/bschiffthaler/sed", task) + + # Known `container` CLI limitation (v1.2.2): Seqera Containers serve a bare single-platform + # OCI manifest, which Apple Container rejects with "does not list any supported platform". + # If this starts passing, upstream has fixed it and this can move to the successful pulls. + with pytest.raises(DockerError.OtherError): + fetcher.pull_image("community.wave.seqera.io/library/umi-transfer:1.0.0--e5b0c1a65b8173b6", task) + + # Failed pulls must raise, not pass silently + with pytest.raises((DockerError.ImageNotFoundError, DockerError.OtherError)): + fetcher.pull_image("docker.io/bschiffthaler/nothingtopullhere", task) + with pytest.raises((DockerError.InvalidTagError, DockerError.OtherError)): + fetcher.pull_image("ghcr.io/ewels/multiqc:go-rewrite", task) + with pytest.raises(DockerError.OtherError): + fetcher.pull_image("Not:A:Valid//reference", task) + + # + # Tests for 'save_image' + # + @requires_apple_container + @with_temporary_folder + def test_apple_container_save_image_not_pulled(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = self._make_fetcher(tmp_path) + output_path = tmp_path / "not-pulled.tar" + with pytest.raises(DockerError.ImageNotPulledError): + fetcher.save_image( + "this-image-cannot-possibly-be-pulled-to-this-machine:latest", + output_path, + mock.MagicMock(), + ) + assert not output_path.exists() + + # + # Tests for 'pull_and_save_image' + # + @requires_apple_container + @with_temporary_folder + def test_apple_container_pull_and_save_image(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = self._make_fetcher(tmp_path) + output_path = tmp_path / "hello-world.tar" + + progress = fetcher.progress = mock.MagicMock() + + fetcher.pull_and_save_image("hello-world", output_path) + + assert output_path.exists() + assert tarfile.is_tarfile(output_path) + assert "ERROR" not in [record.levelname for record in self._caplog.records] + progress.advance_remote_fetch_task.assert_called_once() + + @requires_apple_container + @with_temporary_folder + def test_apple_container_pull_and_save_image_missing(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = self._make_fetcher(tmp_path) + output_path = tmp_path / "nothingtopullhere.tar" + + progress = fetcher.progress = mock.MagicMock() + + # Errors are logged, not raised, so that one bad image doesn't stop the whole download + fetcher.pull_and_save_image("docker.io/bschiffthaler/nothingtopullhere", output_path) + + assert not output_path.exists() + assert "ERROR" in [record.levelname for record in self._caplog.records] + progress.advance_remote_fetch_task.assert_called_once() + + # + # Tests for 'fetch_containers': end-to-end through the base class and fetch_remote_containers + # + @requires_apple_container + @with_temporary_folder + @mock.patch.object(AppleContainerFetcher, "gather_registries", return_value=set()) + def test_apple_container_fetch_containers(self, tmp_path, mock_gather_registries): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + hide_progress=True, + ) + out_dir = fetcher.get_container_output_dir() + out_dir.mkdir() + + good = "hello-world" + bad = "docker.io/bschiffthaler/nothingtopullhere" + fetcher.fetch_containers([good, bad], [], workflow_directory=tmp_path) + + good_tar = out_dir / fetcher.get_container_filename(good) + assert good_tar.exists() + assert tarfile.is_tarfile(good_tar) + assert not (out_dir / fetcher.get_container_filename(bad)).exists() From 4d11ba80803febcadfe6da47c541d58685829847 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 14:41:52 +0200 Subject: [PATCH 29/32] Fix linting error by removing trailler space --- nf_core/pipelines/create/template_features.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index b7bc338bd6..2a12bb0a5a 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -406,7 +406,7 @@ configurations: default (`process.arch = 'linux/amd64'`, `--platform linux/amd64`), so standard pipelines run out of the box. - For faster emulation, set `--apple_rosetta` to run images through Rosetta. + For faster emulation, set `--apple_rosetta` to run images through Rosetta. This requires Rosetta to be installed on the host. nfcore_pipelines: True custom_pipelines: True From cf223fc737e1f71d8667f32fbc903410e7e32612 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 14:51:20 +0200 Subject: [PATCH 30/32] Make it clear what Rosetta version is required --- nf_core/pipeline-template/docs/usage.md | 2 +- nf_core/pipeline-template/nextflow_schema.json | 4 ++-- nf_core/pipelines/create/template_features.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index caecf955a1..e4a9b629f5 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -166,7 +166,7 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `apple_container` - A generic configuration profile to be used with [Apple container](https://github.com/apple/container) on macOS (Apple silicon) - - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta emulation (requires Rosetta 2 installed on the host. + - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta 2 emulation (requires Rosetta 2 installed on the host. - `apple_container_arm64` - An Apple container profile that provisions native arm64 images from conda via [Wave](https://seqera.io/wave/) instead of emulating amd64. Faster per task when an arm64 build exists, but only works for tools that have an arm64 build available (conda `linux-aarch64` or a multi-arch image). - Can be used on its own (`-profile apple_container_arm64`) or composed after `apple_container` (`-profile apple_container,apple_container_arm64`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 140be69fed..505028a312 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -230,8 +230,8 @@ "apple_rosetta": { "type": "boolean", "fa_icon": "fab fa-apple", - "description": "Use faster Rosetta amd64 emulation in the apple_container profile instead of plain amd64 emulation.", - "help_text": "Only applies to the `apple_container` profile. When `true`, amd64 container images are emulated with Rosetta (faster) instead of the default plain amd64 emulation. Requires Rosetta to be installed on the host.", + "description": "Use faster Rosetta 2 amd64 emulation in the apple_container profile instead of plain amd64 emulation.", + "help_text": "Only applies to the `apple_container` profile. When `true`, amd64 container images are emulated with Rosetta 2 (faster) instead of the default plain amd64 emulation. Requires Rosetta 2 to be installed on the host.", "hidden": true }{% endif %}{% if nf_schema %}, "help": { diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 2a12bb0a5a..31e618532c 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -406,8 +406,8 @@ configurations: default (`process.arch = 'linux/amd64'`, `--platform linux/amd64`), so standard pipelines run out of the box. - For faster emulation, set `--apple_rosetta` to run images through Rosetta. - This requires Rosetta to be installed on the host. + For faster emulation, set `--apple_rosetta` to run images through Rosetta 2. + This requires Rosetta 2 to be installed on the host. nfcore_pipelines: True custom_pipelines: True default: true From 862f7f5292fae592ebd9255b89cd1d17e4f887e5 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 15:25:05 +0200 Subject: [PATCH 31/32] Update snapshot/test --- .github/snapshots/apple_container.nf.test.snap | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/snapshots/apple_container.nf.test.snap b/.github/snapshots/apple_container.nf.test.snap index 5435135a0f..227342853a 100644 --- a/.github/snapshots/apple_container.nf.test.snap +++ b/.github/snapshots/apple_container.nf.test.snap @@ -3,10 +3,10 @@ "content": [ { "FASTQC": { - "fastqc": "0.12.1" + "fastqc": "0.13.0" }, "Workflow": { - "my-prefix/testpipeline": "v1.0.0dev" + "nf-core/testpipeline": "v1.0.0dev" } }, [ @@ -87,11 +87,11 @@ "multiqc/multiqc_plots/svg/fastqc_top_overrepresented_sequences_table.svg", "multiqc/multiqc_report.html", "pipeline_info", - "pipeline_info/testpipeline_software_mqc_versions.yml" + "pipeline_info/nf_core_testpipeline_software_mqc_versions.yml" ], [ "fastqc-status-check-heatmap.txt:md5,0f1975c565a16bf09be08a05c204ded7", - "fastqc_overrepresented_sequences_plot.txt:md5,4b23cea39c4e23deef6b97810bc1ee46", + "fastqc_overrepresented_sequences_plot.txt:md5,4ccee85de50285e09cb9236cdb9ad5e1", "fastqc_per_base_n_content_plot.txt:md5,037692101c0130c72493d3bbfa3afac1", "fastqc_per_base_sequence_quality_plot.txt:md5,bfe735f3e31befe13bdf6761bb297d6e", "fastqc_per_sequence_gc_content_plot_Counts.txt:md5,7108d19c46ef7883e864ba274c457d2e", @@ -101,7 +101,7 @@ "fastqc_sequence_duplication_levels_plot.txt:md5,3cde2db4033f6c64648976d1174db925", "fastqc_sequence_length_distribution_plot.txt:md5,e82b9b14a7e24c0c5f27af97cebb6870", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", - "multiqc_fastqc.txt:md5,1a41c2158adc9947bff9232962f70110", + "multiqc_fastqc.txt:md5,b40bf936b2521dfb491235737669cbd7", "multiqc_general_stats.txt:md5,0b54e4e764665bd57fe0f95216744a78" ] ], @@ -109,6 +109,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2025-06-16T14:29:10.076573" + "timestamp": "2026-03-06T10:45:17.184283" } -} +} \ No newline at end of file From 970b86bc1ad78c11a6f825595715d493e933e8c9 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Tue, 13 Oct 2026 15:30:45 +0200 Subject: [PATCH 32/32] Fix apple_container snapshot --- .github/snapshots/apple_container.nf.test.snap | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/snapshots/apple_container.nf.test.snap b/.github/snapshots/apple_container.nf.test.snap index 227342853a..59e9f0d3e8 100644 --- a/.github/snapshots/apple_container.nf.test.snap +++ b/.github/snapshots/apple_container.nf.test.snap @@ -6,7 +6,7 @@ "fastqc": "0.13.0" }, "Workflow": { - "nf-core/testpipeline": "v1.0.0dev" + "my-prefix/testpipeline": "v1.0.0dev" } }, [ @@ -87,7 +87,7 @@ "multiqc/multiqc_plots/svg/fastqc_top_overrepresented_sequences_table.svg", "multiqc/multiqc_report.html", "pipeline_info", - "pipeline_info/nf_core_testpipeline_software_mqc_versions.yml" + "pipeline_info/testpipeline_software_mqc_versions.yml" ], [ "fastqc-status-check-heatmap.txt:md5,0f1975c565a16bf09be08a05c204ded7", @@ -111,4 +111,4 @@ }, "timestamp": "2026-03-06T10:45:17.184283" } -} \ No newline at end of file +}