From d62ff994f2bede387b321bb79399b536f52ff145 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 3 Sep 2026 08:01:37 +0200 Subject: [PATCH 1/5] Allow downloading a single file by project name without a local checkout --- .gitignore | 1 + mergin/cli.py | 23 ++++++++++--- mergin/client.py | 6 ++-- mergin/client_pull.py | 69 ++++++++++++++++++++++++++++---------- mergin/test/test_client.py | 36 ++++++++++++++++++++ 5 files changed, 111 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index adb34dd..1db6fa6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ deps venv debug.py .vscode/ +.python-version \ No newline at end of file diff --git a/mergin/cli.py b/mergin/cli.py index c20beb9..8ea33de 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -335,17 +335,32 @@ def share(ctx, project): @click.argument("filepath") @click.argument("output") @click.option("--version", help="Project version tag, for example 'v3'") +@click.option( + "--project", + help="Full project name ('/') to download the file directly from the server. " + "If not given, the current directory is used and must be an existing checked out project.", +) @click.pass_context -def download_file(ctx, filepath, output, version): +def download_file(ctx, filepath, output, version, project): """ - Download project file at specified version. `project` needs to be a combination of namespace/project. - If no version is given, the latest will be fetched. + Download project file at specified version. If no version is given, the latest will be fetched. """ mc = ctx.obj["client"] if mc is None: return + if project is None: + # no --project given, so we default to the current directory - make sure that's actually a checked out project + try: + MerginProject(os.getcwd()).project_full_name() + except InvalidProject: + click.secho( + "Current directory is not a Mergin Maps project. Run this command from within a " + "checked out project directory, or pass --project /.", + fg="red", + ) + return try: - job = download_file_async(mc, os.getcwd(), filepath, output, version) + job = download_file_async(mc, project or os.getcwd(), filepath, output, version) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): diff --git a/mergin/client.py b/mergin/client.py index 1555051..6c0a30d 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -1199,7 +1199,7 @@ def download_file(self, project_dir, file_path, output_filename, version=None): """ Download project file at specified version. Get the latest if no version specified. - :param project_dir: project local directory + :param project_dir: project local directory or a full project name ("/") :type project_dir: String :param file_path: relative path of file to download in the project directory :type file_path: String @@ -1401,11 +1401,11 @@ def download_files( """ Download project files at specified version. Get the latest if no version specified. - :param project_dir: project local directory + :param project_dir: project local directory or a full project name ("/") :type project_dir: String :param file_path: List of relative paths of files to download in the project directory :type file_path: List[String] - :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir. + :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir (only valid when project_dir is an existing local checkout). :type output_paths: List[String] :param version: optional version tag for downloaded file :type version: String diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..c4d1c7b 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -22,11 +22,11 @@ import concurrent.futures -from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType +from .common import CHUNK_SIZE, ClientError, DeltaChangeType, InvalidProject, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file -from typing import List, Optional +from .utils import cleanup_tmp_dir, is_versioned_file, save_to_file +from typing import List, Optional, Union # status = download_project_async(...) # @@ -54,7 +54,7 @@ def __init__( update_tasks, download_queue_items, tmp_dir: tempfile.TemporaryDirectory, - mp, + mp: Union[MerginProject, "DownloadScratchContext"], project_info, ): self.project_path = project_path @@ -64,7 +64,7 @@ def __init__( self.update_tasks = update_tasks self.download_queue_items = download_queue_items self.tmp_dir = tmp_dir - self.mp = mp # MerginProject instance + self.mp = mp self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server self.failure_log_file = None # log file, copied from the project directory if download fails @@ -80,6 +80,25 @@ def dump(self): print("--- END ---") +class DownloadScratchContext: + """ + Minimal stand-in for MerginProject, used by download_files_async() when downloading files + directly by project name ("/") without an existing local project checkout. + + Provides only what the shared download job code actually needs from MerginProject. + """ + + def __init__(self, mc, cache_dir: str): + self.log = mc.log + self.cache_dir = cache_dir + # only used by _cleanup_failed_download() to look for a log file + self.dir = cache_dir + + def remove_logging_handler(self): + # no-op: self.log is mc's shared logger, not owned by this throwaway context + pass + + class DownloadQueueItem: """ a piece of data from a project that should be downloaded - it can be either a chunk or it can be a diff. @@ -398,7 +417,7 @@ def __init__( self.download_queue_items = download_queue_items self.latest_version = latest_version - def apply(self, directory, mp): + def apply(self, directory, mp: Union[MerginProject, "DownloadScratchContext"]): """assemble downloaded chunks into a single file""" if self.destination_file is None: @@ -411,14 +430,14 @@ def apply(self, directory, mp): os.makedirs(file_dir, exist_ok=True) # ignore check if we download not-latest version of gpkg file (possibly reconstructed on server on demand) - check_size = self.latest_version or not mp.is_versioned_file(self.file_path) + check_size = self.latest_version or not is_versioned_file(self.file_path) # merge chunks together (and delete them afterwards) file_to_merge = DownloadFile(dest_file_path, self.download_queue_items, check_size) file_to_merge.from_chunks() # Make a copy of the file to meta dir only if there is no user-specified path for the file. - # destination_file is None for full project download and takes a meaningful value for a single file download. - if mp.is_versioned_file(self.file_path) and self.destination_file is None: + # destination_file is None for full project download and takes a meaningful value for a single file download + if self.destination_file is None and is_versioned_file(self.file_path): mp.geodiff.make_copy_sqlite(mp.fpath(self.file_path), mp.fpath_meta(self.file_path)) @@ -902,9 +921,30 @@ def download_files_async( """ Starts background download project files at specified version. Returns handle to the pending download. + + `project_dir` can either be an existing local project directory (previously fetched with + download_project()), or a full project name ("/") to download files + directly from the server without needing a local checkout. In the latter case, `output_paths` + must be provided explicitly, as there is no project directory to place files into by default. """ - mp = MerginProject(project_dir) - project_path = mp.project_full_name() + # temporary directory to stage downloaded chunks in + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + + mp: Union[MerginProject, "DownloadScratchContext"] + try: + mp = MerginProject(project_dir) + project_path = mp.project_full_name() + except InvalidProject: + # project_dir is not an existing local checkout - treat it as a full project name + # ("/") and download straight from the server instead + if output_paths is None: + cleanup_tmp_dir(mc, tmp_dir) + raise ClientError( + "output_paths must be provided when downloading files without an existing local project checkout" + ) + project_path = project_dir + mp = DownloadScratchContext(mc, tmp_dir.name) + ver_info = f"at version {version}" if version is not None else "at latest version" mp.log.info(f"Getting [{', '.join(file_paths)}] {ver_info}") latest_proj_info = mc.project_info(project_path) @@ -914,9 +954,6 @@ def download_files_async( project_info = latest_proj_info mp.log.info(f"Got project info. version {project_info['version']}") - # set temporary directory for download - tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") - if output_paths is None: output_paths = [] for file in file_paths: @@ -991,6 +1028,4 @@ def download_files_finalize(job: DownloadJob): for task in job.update_tasks: task.apply(job.tmp_dir, job.mp) - # Remove temporary download directory - if job.tmp_dir is not None and os.path.exists(job.tmp_dir.name): - cleanup_tmp_dir(job.mp, job.tmp_dir) + cleanup_tmp_dir(job.mp, job.tmp_dir) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index bd987d0..d84c7b8 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1354,6 +1354,42 @@ def test_download_file(mc): mc.download_file(project_dir, f_updated, f_downloaded, version="v5") +def test_download_file_without_checkout(mc): + """Test downloading a single file directly by project name, without an existing local checkout.""" + test_project = "test_download_file_without_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + f_updated = "base.gpkg" + + create_versioned_project(mc, test_project, project_dir, f_updated) + + # download straight from the server by "workspace/project" name into a fresh directory + # that has never been used as a project checkout + download_dir = os.path.join(TMP_DIR, test_project + "_no_checkout") + remove_folders([download_dir]) + os.makedirs(download_dir, exist_ok=True) + f_downloaded = os.path.join(download_dir, f_updated) + + expected_content = "inserted_1_A.gpkg" + mc.download_file(project, f_updated, f_downloaded, version="v2") + expected = os.path.join(TEST_DATA_DIR, expected_content) + assert check_gpkg_same_content(MerginProject(project_dir), f_downloaded, expected) + assert not os.path.exists(os.path.join(download_dir, ".mergin")) + + # output_paths must be provided explicitly when there is no local checkout + with pytest.raises(ClientError, match="output_paths must be provided"): + mc.download_files(project, [f_updated]) + + # non-existent file in an existing project - same error as with a local checkout + with pytest.raises(ClientError, match=r"No \[does_not_exist\.gpkg\] exists at version v2"): + mc.download_file(project, "does_not_exist.gpkg", f_downloaded, version="v2") + + # non-existent / inaccessible project should fail clearly too + nonexistent_project = create_project_path("this_project_does_not_exist", mc) + with pytest.raises(ClientError): + mc.download_file(nonexistent_project, f_updated, f_downloaded) + + def test_download_diffs(mc): """Test download diffs for a project file between specified project versions.""" test_project = "test_download_diffs" From 27311edf27369d9896a52e9b5eaf141b1bf67940 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 3 Sep 2026 15:03:03 +0200 Subject: [PATCH 2/5] Add sparse checkout support: filter downloaded/pushed files by include/exclude pattern --- mergin/cli.py | 8 +- mergin/client.py | 17 +- mergin/client_pull.py | 25 ++- mergin/client_push.py | 10 +- mergin/merginproject.py | 14 +- mergin/test/test_client.py | 265 +++++++++++++++++++++++++++++ mergin/test/test_mergin_project.py | 1 + mergin/test/test_utils.py | 80 +++++++++ mergin/utils.py | 52 +++++- 9 files changed, 462 insertions(+), 10 deletions(-) create mode 100644 mergin/test/test_utils.py diff --git a/mergin/cli.py b/mergin/cli.py index c20beb9..ed9ad7c 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -248,16 +248,20 @@ def list_projects(ctx, name, namespace, order_params): @click.argument("project") @click.argument("directory", type=click.Path(), required=False) @click.option("--version", default=None, help="Version of project to download") +@click.option("--include", multiple=True, help="Only download files matching this pattern, e.g. '*.gpkg'") +@click.option("--exclude", multiple=True, help="Skip files matching this pattern, e.g. 'media/*'") @click.pass_context -def download(ctx, project, directory, version): +def download(ctx, project, directory, version, include, exclude): """Download last version of mergin project.""" mc = ctx.obj["client"] if mc is None: return + if include and exclude: + raise click.UsageError("--include and --exclude cannot be used together") directory = directory or os.path.basename(project) click.echo("Downloading into {}".format(directory)) try: - job = download_project_async(mc, project, directory, version) + job = download_project_async(mc, project, directory, version, include=include, exclude=exclude) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): diff --git a/mergin/client.py b/mergin/client.py index 1555051..efe1c8a 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -63,6 +63,7 @@ from .utils import DateTimeEncoder, get_versions_with_file_changes, int_version, is_version_acceptable from .utils import ( DateTimeEncoder, + filter_files, get_versions_with_file_changes, int_version, is_version_acceptable, @@ -902,7 +903,7 @@ def project_versions(self, project_path, since=1, to=None): filtered_versions = list(filter(lambda v: (num_since <= int_version(v["name"]) <= num_to), versions)) return filtered_versions - def download_project(self, project_path, directory, version=None): + def download_project(self, project_path, directory, version=None, include=None, exclude=None): """ Download project into given directory. If version is not specified, latest version is downloaded @@ -914,8 +915,16 @@ def download_project(self, project_path, directory, version=None): :param version: Project version to download, e.g. v42 :type version: String + + :param include: Optional list of glob patterns (matched against each file's project path, e.g. + "media/*" or "*.gpkg") - only matching files are downloaded. + :type include: List[String] + + :param exclude: Optional list of glob patterns - matching files are skipped. Mutually exclusive + with include. + :type exclude: List[String] """ - job = download_project_async(self, project_path, directory, version) + job = download_project_async(self, project_path, directory, version, include=include, exclude=exclude) download_project_wait(job) download_project_finalize(job) @@ -1158,6 +1167,10 @@ def project_status(self, directory): server_info = self.project_info(mp.project_full_name(), since=mp.version()) pull_changes = mp.get_pull_changes(server_info.get("files", []), server_info.get("version")) + # on a sparse checkout, don't report excluded files as pending server changes - + # they were never meant to be pulled in the first place + file_filter = mp.file_filter() + pull_changes = {change_type: filter_files(files, **file_filter) for change_type, files in pull_changes.items()} push_changes = mp.get_push_changes() push_changes_summary = mp.get_list_of_push_changes(push_changes) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..6805dd2 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file +from .utils import cleanup_tmp_dir, filter_files, path_matches_filter, save_to_file, validates_file_filter from typing import List, Optional # status = download_project_async(...) @@ -242,10 +242,15 @@ def _cleanup_failed_download(mergin_project: MerginProject = None): return dest_path -def download_project_async(mc, project_path, directory, project_version=None): +@validates_file_filter +def download_project_async(mc, project_path, directory, project_version=None, include=None, exclude=None): """ Starts project download in background and returns handle to the pending project download. Using that object it is possible to watch progress or cancel the ongoing work. + + `include`/`exclude` are optional lists of glob patterns (matched against each file's project + path, e.g. "media/*" or "*.gpkg") to only download a subset of the project's files. They are + mutually exclusive. """ if "/" not in project_path: @@ -276,6 +281,11 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info(f"got project info. version {version}") + # keep only the files matching the filter (if any) + project_info["files"] = filter_files(project_info["files"], include=include, exclude=exclude) + if include or exclude: + project_info["file_filter"] = {"include": include, "exclude": exclude} + # prepare download update_tasks = [] # stuff to do at the end of download for file in project_info["files"]: @@ -525,6 +535,9 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: mp.log.info("--- pull aborted") raise + file_filter = mp.file_filter() + delta.changes = [c for c in delta.changes if path_matches_filter(c.path, **file_filter)] + mp.log.info(f"got project versions: local version {local_version} / server version {server_version}") if local_version == server_version: @@ -748,6 +761,14 @@ def pull_project_finalize(job: PullJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise ClientError("Failed to apply pull actions: " + str(e)) + file_filter = job.mp.file_filter() + job.project_info["files"] = filter_files(job.project_info["files"], **file_filter) + # keep the sparse checkout: re-apply the filter this project was downloaded with, + # since job.project_info is a fresh, unfiltered response from the server + # and update_metadata() replaces the whole metadata dict rather than merging into it + if file_filter["include"] or file_filter["exclude"]: + job.project_info["file_filter"] = file_filter + job.mp.update_metadata(job.project_info) if job.mp.has_unfinished_pull(): diff --git a/mergin/client_push.py b/mergin/client_push.py index 831b59b..1c9d3d7 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -34,7 +34,7 @@ ) from .merginproject import MerginProject, pygeodiff from .editor import filter_changes -from .utils import get_data_checksum, cleanup_tmp_dir +from .utils import get_data_checksum, cleanup_tmp_dir, filter_files POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -458,6 +458,14 @@ def push_project_finalize(job: UploadJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise err + # keep the sparse checkout: re-apply the filter this project was + # downloaded with, since job.server_resp is a fresh, unfiltered response from the server + # and update_metadata() replaces the whole metadata dict rather than merging into it + file_filter = job.mp.file_filter() + job.server_resp["files"] = filter_files(job.server_resp["files"], **file_filter) + if file_filter["include"] or file_filter["exclude"]: + job.server_resp["file_filter"] = file_filter + job.mp.update_metadata(job.server_resp) try: job.mp.apply_push_changes(asdict(job.changes)) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 12d798f..b4544e7 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -24,6 +24,7 @@ unique_path_name, conflicted_copy_file_name, edit_conflict_file_name, + filter_files, ) from .local_changes import FileChange @@ -215,6 +216,13 @@ def files(self) -> list: self._read_metadata() return self._metadata["files"] + def file_filter(self) -> dict: + """ + Returns the include/exclude file filter this project was downloaded with, as a dict with "include" and "exclude" keys. + """ + self._read_metadata() + return self._metadata.get("file_filter", {"include": None, "exclude": None}) + @property def metadata(self) -> dict: """Gets raw access to metadata. Kept only for backwards compatibility and will be removed.""" @@ -566,7 +574,8 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: :rtype: List[ProjectDeltaItem] """ result = [] - changes = self.compare_file_sets(self.files(), self.inspect_files()) + current_files = filter_files(self.inspect_files(), **self.file_filter()) + changes = self.compare_file_sets(self.files(), current_files) added = changes.get("added", []) removed = changes.get("removed", []) updated = changes.get("updated", []) @@ -655,7 +664,8 @@ def get_push_changes(self): :returns: changes metadata for files to be pushed to server :rtype: dict """ - changes = self.compare_file_sets(self.files(), self.inspect_files()) + current_files = filter_files(self.inspect_files(), **self.file_filter()) + changes = self.compare_file_sets(self.files(), current_files) # do checkpoint to push changes from wal file to gpkg for file in changes["added"] + changes["updated"]: size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index bd987d0..93e9153 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1,4 +1,5 @@ import hashlib +import inspect import json import logging import os @@ -1153,6 +1154,270 @@ def test_download_versions(mc): mc.download_project(project, project_dir_v3, "v3") +def test_download_project_with_filter(mc): + """Test downloading a project with include/exclude filters, and that they're mutually exclusive.""" + test_project = "test_download_project_filter" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + include_dir = os.path.join(TMP_DIR, test_project + "_include") + exclude_dir = os.path.join(TMP_DIR, test_project + "_exclude") + conflict_dir = os.path.join(TMP_DIR, test_project + "_conflict") + + cleanup(mc, project, [project_dir, include_dir, exclude_dir, conflict_dir]) + shutil.copytree(TEST_DATA_DIR, project_dir) + mc.create_project_and_push(project, project_dir) + + # include filter: only matching files are fetched + mc.download_project(project, include_dir, include=["*.gpkg"]) + + downloaded_files = set() + for root, _, files in os.walk(include_dir): + if ".mergin" in root.split(os.sep): + continue + for f in files: + rel = os.path.relpath(os.path.join(root, f), include_dir) + downloaded_files.add(rel.replace(os.sep, "/")) + + assert downloaded_files + assert all(f.endswith(".gpkg") for f in downloaded_files) + assert "test.qgs" not in downloaded_files + assert "test.txt" not in downloaded_files + + mp = MerginProject(include_dir) + assert all(f["path"].endswith(".gpkg") for f in mp.files()) + assert mp.file_filter() == {"include": ["*.gpkg"], "exclude": None} + + # exclude filter: matching files are skipped + mc.download_project(project, exclude_dir, exclude=["test_dir/*"]) + + assert os.path.exists(os.path.join(exclude_dir, "base.gpkg")) + assert not os.path.exists(os.path.join(exclude_dir, "test_dir")) + + mp = MerginProject(exclude_dir) + assert not any(f["path"].startswith("test_dir/") for f in mp.files()) + assert mp.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + + # include and exclude cannot be combined + with pytest.raises(ClientError, match="Cannot use both include and exclude"): + mc.download_project(project, conflict_dir, include=["*.gpkg"], exclude=["*.txt"]) + assert not os.path.exists(conflict_dir) + + +def test_sparse_checkout_filter_is_immutable(mc): + """Once a directory has been downloaded with a filter, there is no way to change that + filter in place - the only way to get a different filter is to check out into a fresh + directory. + """ + test_project = "test_sparse_checkout_immutable" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + shutil.copytree(TEST_DATA_DIR, project_dir) + mc.create_project_and_push(project, project_dir) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + original_filter = MerginProject(sparse_dir).file_filter() + + # trying to download again into the same directory - with the same or a different filter - + # must fail without touching anything, regardless of what filter (if any) is requested + for kwargs in ({"exclude": ["test_dir/*"]}, {"include": ["*.gpkg"]}, {}): + with pytest.raises(ClientError, match="Project directory already exists"): + mc.download_project(project, sparse_dir, **kwargs) + + assert MerginProject(sparse_dir).file_filter() == original_filter + + # pull_project()/push_project() take no filter arguments at all - there is no API through + # which a different filter could be supplied for an existing checkout + assert "include" not in inspect.signature(mc.pull_project).parameters + assert "include" not in inspect.signature(mc.push_project).parameters + + +def test_pull_on_sparse_checkout(mc): + """A sparse checkout keeps respecting its filter across subsequent pulls: excluded files + stay ignored even as the server moves ahead, and a genuine local-vs-server conflict on an + included file is still detected and resolved correctly. + """ + test_project = "test_pull_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + sparse_version_before = MerginProject(sparse_dir).version() + + # change an excluded file on the server, by pushing from the reference (full) checkout + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change that only affects an excluded file") + mc.push_project(project_dir) + server_version = mc.project_info(project)["version"] + assert server_version != sparse_version_before + + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == server_version + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + + # now a genuine conflict: edit an included file locally, but don't push it yet + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + + # meanwhile a conflicting edit to the same file gets pushed from the reference (full) checkout + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(project_dir, "base.gpkg")) + mc.push_project(project_dir) + server_version = mc.project_info(project)["version"] + assert server_version != mp_sparse.version() + + # pulling the sparse checkout now must detect the conflict (not silently drop the local + # edit, not crash, and not leave the sparse checkout in a broken state) + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == server_version + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) # filter still respected + conflict_files = [f for f in os.listdir(sparse_dir) if "conflicted copy" in f] + assert conflict_files, "expected a conflicted copy of base.gpkg to be created" + + +def test_push_on_sparse_checkout(mc): + """Pushing from a sparse checkout must never touch the files it never downloaded: not a + no-op push (must not delete excluded files it's not tracking), not a real push of an + included file's edit (must not wipe the persisted filter from local metadata. + """ + test_project = "test_push_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # no local edits - must be a no-op, not a deletion of excluded files + mc.push_project(sparse_dir) + + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + # a real push of an included file's edit must not lose the persisted filter afterwards + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + # manually add a file outside the filter's scope - it must not get pushed either + os.makedirs(os.path.join(sparse_dir, "test_dir"), exist_ok=True) + with open(os.path.join(sparse_dir, "test_dir", "new_stray.txt"), "w") as f: + f.write("should not be pushed") + + mc.push_project(sparse_dir) + + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/new_stray.txt" not in server_files + + # a delete-only push goes through a different shortcut code path server-side + # it must also keep the filter intact and leave excluded files alone + # old_metadata.json already present on server and not affected by filter + os.remove(os.path.join(sparse_dir, "old_metadata.json")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "old_metadata.json" not in server_files + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + +def test_sparse_checkout_pull_push_v1_api(mc): + """Same filter-persistence guarantees as test_pull_on_sparse_checkout/test_push_on_sparse_checkout, + but forcing the legacy v1 pull/push code paths. + """ + server_features = mc.server_features() + mc._server_features = {"v2_pull_enabled": False, "v2_push_enabled": False} + + test_project = "test_sparse_checkout_v1_api" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # change an excluded file on the server - a v1 pull must still skip it + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change to an excluded file, v1 api") + mc.push_project(project_dir) + + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == mc.project_info(project)["version"] + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + + # a real edit to an included file, pushed via v1 - filter must survive it + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + mc._server_features = server_features + + +def test_project_status_on_sparse_checkout(mc): + """`status` must not report excluded files as pending server changes - + they were deliberately never meant to be pulled, so they shouldn't show up as if a pull + were needed to fetch them. + """ + test_project = "test_status_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # change both an excluded and an included file on the server + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change to an excluded file") + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(project_dir, "base.gpkg")) + mc.push_project(project_dir) + + pull_changes, _, _ = mc.project_status(sparse_dir) + + changed_paths = {f["path"] for files in pull_changes.values() for f in files} + assert "test_dir/test2.txt" not in changed_paths + # a real, included change must still be reported + assert "base.gpkg" in changed_paths + + def test_paginated_project_list(mc): """Test the new endpoint for projects list with pagination, ordering etc.""" test_projects = dict() diff --git a/mergin/test/test_mergin_project.py b/mergin/test/test_mergin_project.py index 97fe554..b6879ea 100644 --- a/mergin/test/test_mergin_project.py +++ b/mergin/test/test_mergin_project.py @@ -186,6 +186,7 @@ def test_get_local_delta(): # Mock files() to return origin info for version lookup mp.files = lambda: [] mp.inspect_files = lambda: [] # Dummy return + mp.file_filter = lambda: {"include": None, "exclude": None} # no sparse checkout filter in this test # check if geopackage is updated (is_open) but missing - geodiff lib error, than updated file is reported mock_changes = { diff --git a/mergin/test/test_utils.py b/mergin/test/test_utils.py new file mode 100644 index 0000000..973db25 --- /dev/null +++ b/mergin/test/test_utils.py @@ -0,0 +1,80 @@ +import pytest + +from ..utils import path_matches_filter, filter_files +from ..common import ClientError + + +@pytest.mark.parametrize( + "path, include, exclude, expected", + [ + # no filter at all -> everything kept + pytest.param("anything/at/all.gpkg", None, None, True, id="no-filter"), + pytest.param("data.gpkg", [], None, True, id="empty-include-list"), + pytest.param("data.gpkg", None, [], True, id="empty-exclude-list"), + # basic include/exclude + pytest.param("data.gpkg", ["*.gpkg"], None, True, id="include-match"), + pytest.param("data.txt", ["*.gpkg"], None, False, id="include-no-match"), + pytest.param("media/photo.jpg", None, ["media/*"], False, id="exclude-match"), + pytest.param("data.gpkg", None, ["media/*"], True, id="exclude-no-match"), + # subfolders: fnmatch's '*' crosses '/', so it reaches arbitrarily deep + pytest.param("media/photo.jpg", None, ["media/*"], False, id="subfolder-direct-child"), + pytest.param("media/sub/deep/photo.jpg", None, ["media/*"], False, id="subfolder-deeply-nested"), + pytest.param("layer.gpkg", ["*.gpkg"], None, True, id="extension-pattern-at-root"), + pytest.param("data/nested/deep/layer.gpkg", ["*.gpkg"], None, True, id="extension-pattern-at-any-depth"), + # patterns still anchor to the *full* path, not just the basename + pytest.param("nested/media/photo.jpg", None, ["media/*"], True, id="not-anchored-to-basename-kept"), + pytest.param("nested/media/photo.jpg", None, ["*/media/*"], False, id="leading-star-catches-nested-media"), + pytest.param("media/photo.jpg", None, ["*/media/*"], True, id="leading-star-misses-root-level-media"), + # case sensitivity: fnmatchcase, not fnmatch - always case-sensitive, any OS + pytest.param("data.GPKG", ["*.gpkg"], None, False, id="case-mismatch-in-path"), + pytest.param("data.gpkg", ["*.GPKG"], None, False, id="case-mismatch-in-pattern"), + pytest.param("Media/photo.jpg", None, ["media/*"], True, id="case-mismatch-in-directory-kept"), + # lists of patterns: a path matches if it matches ANY pattern in the list (OR) + pytest.param("data.gpkg", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-1st-matches"), + pytest.param("map.qgz", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-2nd-matches"), + pytest.param("project.qgs", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-3rd-matches"), + pytest.param("readme.txt", ["*.gpkg", "*.qgz", "project.qgs"], None, False, id="include-list-none-match"), + pytest.param("media/photo.jpg", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-1st-matches"), + pytest.param("scratch.tmp", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-2nd-matches"), + pytest.param("data.gpkg-wal", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-3rd-matches"), + pytest.param("data.gpkg", None, ["media/*", "*.tmp", "*-wal"], True, id="exclude-list-none-match"), + ], +) +def test_path_matches_filter(path, include, exclude, expected): + assert path_matches_filter(path, include=include, exclude=exclude) is expected + + +@pytest.mark.parametrize( + "include, exclude, expected_paths", + [ + pytest.param(None, None, {"a.gpkg", "b.qgz", "c.txt", "media/d.gpkg"}, id="no-filter"), + pytest.param(["*.gpkg", "*.qgz"], None, {"a.gpkg", "b.qgz", "media/d.gpkg"}, id="include-list"), + pytest.param(None, ["media/*"], {"a.gpkg", "b.qgz", "c.txt"}, id="exclude-subfolder"), + ], +) +def test_filter_files(include, exclude, expected_paths): + files = [{"path": p} for p in ["a.gpkg", "b.qgz", "c.txt", "media/d.gpkg"]] + result = filter_files(files, include=include, exclude=exclude) + assert {f["path"] for f in result} == expected_paths + + +def test_filter_files_keeps_matching_dicts_as_is(): + """filter_files() passes matching dicts through unchanged""" + files = [ + {"path": "project.gpkg", "size": 100}, + {"path": "media/photo.jpg", "size": 999}, + ] + + result = filter_files(files, exclude=["media/*"]) + + assert result == [{"path": "project.gpkg", "size": 100}] + assert result[0] is files[0] + + +def test_filter_files_raises_on_mutually_exclusive_args(): + """Unlike path_matches_filter, filter_files() is a public entry point (decorated with + @validates_file_filter) and does enforce that include/exclude are mutually exclusive. + """ + files = [{"path": "a.gpkg"}] + with pytest.raises(ClientError, match="Cannot use both include and exclude"): + filter_files(files, include=["*.gpkg"], exclude=["*.txt"]) diff --git a/mergin/utils.py b/mergin/utils.py index 91796f3..1b7a234 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -2,13 +2,16 @@ import io import json import hashlib +import fnmatch +import functools +import inspect import re import sqlite3 from datetime import datetime from pathlib import Path import tempfile from enum import Enum -from typing import Optional, Type, Union, ByteString +from typing import List, Optional, Type, Union, ByteString from .common import ClientError @@ -280,6 +283,53 @@ def is_mergin_config(path: str) -> bool: return filename == "mergin-config.json" +def validates_file_filter(func): + """ + Marks a function as accepting an `include`/`exclude` glob-filter signature, and validates + those arguments (mutually exclusive) before every call - regardless of whether the caller + passed them positionally or by keyword. + """ + signature = inspect.signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + bound_args = signature.bind_partial(*args, **kwargs) + if bound_args.arguments.get("include") and bound_args.arguments.get("exclude"): + raise ClientError("Cannot use both include and exclude filters at the same time") + return func(*args, **kwargs) + + return wrapper + + +def path_matches_filter(path: str, include: List[str] = None, exclude: List[str] = None) -> bool: + """ + Returns whether `path` should be kept under a sparse-checkout style include/exclude filter. + + With `include`, only paths matching at least one glob pattern are kept. With `exclude`, + paths matching at least one pattern are dropped. With neither given, every path is kept. + """ + if include: + return any(fnmatch.fnmatchcase(path, pattern) for pattern in include) + if exclude: + return not any(fnmatch.fnmatchcase(path, pattern) for pattern in exclude) + return True + + +@validates_file_filter +def filter_files(files: List[dict], include: List[str] = None, exclude: List[str] = None) -> List[dict]: + """ + Keep only files matching a sparse-checkout style filter. + + :param files: list of file metadata dicts, each with a 'path' key + :param include: glob patterns - only matching files are kept + :param exclude: glob patterns - matching files are dropped + :returns: filtered list of file metadata dicts + + .. seealso:: path_matches_filter + """ + return [f for f in files if path_matches_filter(f["path"], include=include, exclude=exclude)] + + def bytes_to_human_size(bytes: int): """ Convert bytes to human readable size From b664e228a4af604f485ec5c9a06ce6c1289d530c Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 10 Sep 2026 07:47:52 +0200 Subject: [PATCH 3/5] Simplify sparse checkout file handling --- mergin/client_pull.py | 7 ++++--- mergin/merginproject.py | 16 ++++++++++------ mergin/test/test_utils.py | 10 +++++----- mergin/utils.py | 38 ++++++++------------------------------ 4 files changed, 27 insertions(+), 44 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 6805dd2..3366bb4 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, filter_files, path_matches_filter, save_to_file, validates_file_filter +from .utils import cleanup_tmp_dir, filter_files, is_path_in_scope, save_to_file from typing import List, Optional # status = download_project_async(...) @@ -242,7 +242,6 @@ def _cleanup_failed_download(mergin_project: MerginProject = None): return dest_path -@validates_file_filter def download_project_async(mc, project_path, directory, project_version=None, include=None, exclude=None): """ Starts project download in background and returns handle to the pending project download. @@ -253,6 +252,8 @@ def download_project_async(mc, project_path, directory, project_version=None, in mutually exclusive. """ + if include and exclude: + raise ClientError("Cannot use both include and exclude filters at the same time") if "/" not in project_path: raise ClientError("Project name needs to be fully qualified, e.g. /") if os.path.exists(directory): @@ -536,7 +537,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: raise file_filter = mp.file_filter() - delta.changes = [c for c in delta.changes if path_matches_filter(c.path, **file_filter)] + delta.changes = [c for c in delta.changes if is_path_in_scope(c.path, **file_filter)] mp.log.info(f"got project versions: local version {local_version} / server version {server_version}") diff --git a/mergin/merginproject.py b/mergin/merginproject.py index b4544e7..3058a45 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -25,6 +25,7 @@ conflicted_copy_file_name, edit_conflict_file_name, filter_files, + is_path_in_scope, ) from .local_changes import FileChange @@ -212,9 +213,10 @@ def version(self) -> str: return self._metadata["version"] def files(self) -> list: - """Returns project's list of files (each file being a dictionary)""" + """Returns project's list of files (each file being a dictionary), scoped to this + project's file_filter() if one is set (sparse checkout).""" self._read_metadata() - return self._metadata["files"] + return filter_files(self._metadata["files"], **self.file_filter()) def file_filter(self) -> dict: """ @@ -312,10 +314,12 @@ def ignore_file(self, file): def inspect_files(self): """ Inspect files in project directory and return metadata. + Only files matching this project's file_filter() are included. :returns: metadata for files in project directory in server required format :rtype: list[dict] """ + file_filter = self.file_filter() files_meta = [] for root, dirs, files in os.walk(self.dir, topdown=True): dirs[:] = [d for d in dirs if d not in [".mergin"]] @@ -326,6 +330,8 @@ def inspect_files(self): abs_path = os.path.abspath(os.path.join(root, file)) rel_path = os.path.relpath(abs_path, start=self.dir) proj_path = "/".join(rel_path.split(os.path.sep)) # we need posix path + if not is_path_in_scope(proj_path, **file_filter): + continue files_meta.append( { "path": proj_path, @@ -574,8 +580,7 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: :rtype: List[ProjectDeltaItem] """ result = [] - current_files = filter_files(self.inspect_files(), **self.file_filter()) - changes = self.compare_file_sets(self.files(), current_files) + changes = self.compare_file_sets(self.files(), self.inspect_files()) added = changes.get("added", []) removed = changes.get("removed", []) updated = changes.get("updated", []) @@ -664,8 +669,7 @@ def get_push_changes(self): :returns: changes metadata for files to be pushed to server :rtype: dict """ - current_files = filter_files(self.inspect_files(), **self.file_filter()) - changes = self.compare_file_sets(self.files(), current_files) + changes = self.compare_file_sets(self.files(), self.inspect_files()) # do checkpoint to push changes from wal file to gpkg for file in changes["added"] + changes["updated"]: size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log) diff --git a/mergin/test/test_utils.py b/mergin/test/test_utils.py index 973db25..481fa49 100644 --- a/mergin/test/test_utils.py +++ b/mergin/test/test_utils.py @@ -1,6 +1,6 @@ import pytest -from ..utils import path_matches_filter, filter_files +from ..utils import is_path_in_scope, filter_files from ..common import ClientError @@ -40,8 +40,8 @@ pytest.param("data.gpkg", None, ["media/*", "*.tmp", "*-wal"], True, id="exclude-list-none-match"), ], ) -def test_path_matches_filter(path, include, exclude, expected): - assert path_matches_filter(path, include=include, exclude=exclude) is expected +def test_is_path_in_scope(path, include, exclude, expected): + assert is_path_in_scope(path, include=include, exclude=exclude) is expected @pytest.mark.parametrize( @@ -72,8 +72,8 @@ def test_filter_files_keeps_matching_dicts_as_is(): def test_filter_files_raises_on_mutually_exclusive_args(): - """Unlike path_matches_filter, filter_files() is a public entry point (decorated with - @validates_file_filter) and does enforce that include/exclude are mutually exclusive. + """Unlike is_path_in_scope, filter_files() is the validating entry point and + enforces that include/exclude are mutually exclusive. """ files = [{"path": "a.gpkg"}] with pytest.raises(ClientError, match="Cannot use both include and exclude"): diff --git a/mergin/utils.py b/mergin/utils.py index 1b7a234..85c2cca 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -3,8 +3,6 @@ import json import hashlib import fnmatch -import functools -import inspect import re import sqlite3 from datetime import datetime @@ -283,30 +281,14 @@ def is_mergin_config(path: str) -> bool: return filename == "mergin-config.json" -def validates_file_filter(func): - """ - Marks a function as accepting an `include`/`exclude` glob-filter signature, and validates - those arguments (mutually exclusive) before every call - regardless of whether the caller - passed them positionally or by keyword. - """ - signature = inspect.signature(func) - - @functools.wraps(func) - def wrapper(*args, **kwargs): - bound_args = signature.bind_partial(*args, **kwargs) - if bound_args.arguments.get("include") and bound_args.arguments.get("exclude"): - raise ClientError("Cannot use both include and exclude filters at the same time") - return func(*args, **kwargs) - - return wrapper - - -def path_matches_filter(path: str, include: List[str] = None, exclude: List[str] = None) -> bool: +def is_path_in_scope(path: str, include: List[str] = None, exclude: List[str] = None) -> bool: """ Returns whether `path` should be kept under a sparse-checkout style include/exclude filter. With `include`, only paths matching at least one glob pattern are kept. With `exclude`, paths matching at least one pattern are dropped. With neither given, every path is kept. + + Assumes include/exclude were already validated as mutually exclusive by the caller. """ if include: return any(fnmatch.fnmatchcase(path, pattern) for pattern in include) @@ -315,19 +297,15 @@ def path_matches_filter(path: str, include: List[str] = None, exclude: List[str] return True -@validates_file_filter def filter_files(files: List[dict], include: List[str] = None, exclude: List[str] = None) -> List[dict]: """ - Keep only files matching a sparse-checkout style filter. - - :param files: list of file metadata dicts, each with a 'path' key - :param include: glob patterns - only matching files are kept - :param exclude: glob patterns - matching files are dropped - :returns: filtered list of file metadata dicts + Keep only files (dict with 'path' key) matching a sparse-checkout style filter. - .. seealso:: path_matches_filter + .. seealso:: is_path_in_scope """ - return [f for f in files if path_matches_filter(f["path"], include=include, exclude=exclude)] + if include and exclude: + raise ClientError("Cannot use both include and exclude filters at the same time") + return [f for f in files if is_path_in_scope(f["path"], include=include, exclude=exclude)] def bytes_to_human_size(bytes: int): From 4767c67770c747ebfaf2bfc8261ce4511e97bc59 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 10 Sep 2026 10:14:57 +0200 Subject: [PATCH 4/5] Separate download single file to dedicated API call --- mergin/cli.py | 26 +++++++++-------- mergin/client.py | 25 +++++++++++++++-- mergin/client_pull.py | 57 +++++++++++++++++++++++--------------- mergin/test/test_client.py | 12 ++++---- 4 files changed, 76 insertions(+), 44 deletions(-) diff --git a/mergin/cli.py b/mergin/cli.py index 8ea33de..485d16b 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -29,6 +29,7 @@ download_project_cancel, download_file_async, download_file_finalize, + download_project_file_async, download_project_finalize, download_project_is_running, ) @@ -348,19 +349,20 @@ def download_file(ctx, filepath, output, version, project): mc = ctx.obj["client"] if mc is None: return - if project is None: - # no --project given, so we default to the current directory - make sure that's actually a checked out project - try: - MerginProject(os.getcwd()).project_full_name() - except InvalidProject: - click.secho( - "Current directory is not a Mergin Maps project. Run this command from within a " - "checked out project directory, or pass --project /.", - fg="red", - ) - return try: - job = download_file_async(mc, project or os.getcwd(), filepath, output, version) + if project is not None: + job = download_project_file_async(mc, project, filepath, output, version) + else: + try: + MerginProject(os.getcwd()).project_full_name() + except InvalidProject: + click.secho( + "Current directory is not a Mergin Maps project. Run this command from within a " + "checked out project directory, or pass --project /.", + fg="red", + ) + return + job = download_file_async(mc, os.getcwd(), filepath, output, version) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): diff --git a/mergin/client.py b/mergin/client.py index 6c0a30d..17a616a 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -46,6 +46,7 @@ download_file_async, download_files_async, download_files_finalize, + download_project_file_async, download_diffs_async, download_project_finalize, download_project_wait, @@ -1199,7 +1200,7 @@ def download_file(self, project_dir, file_path, output_filename, version=None): """ Download project file at specified version. Get the latest if no version specified. - :param project_dir: project local directory or a full project name ("/") + :param project_dir: project local directory :type project_dir: String :param file_path: relative path of file to download in the project directory :type file_path: String @@ -1212,6 +1213,24 @@ def download_file(self, project_dir, file_path, output_filename, version=None): pull_project_wait(job) download_file_finalize(job) + def download_project_file(self, project_path, file_path, output_filename, version=None): + """ + Download a single project file at specified version directly from the server, without + needing an existing local project checkout. + + :param project_path: full project name ("/") + :type project_path: String + :param file_path: relative path of file to download in the project directory + :type file_path: String + :param output_filename: full destination path for saving the downloaded file + :type output_filename: String + :param version: optional version tag for downloaded file + :type version: String + """ + job = download_project_file_async(self, project_path, file_path, output_filename, version=version) + pull_project_wait(job) + download_file_finalize(job) + def get_file_diff(self, project_dir, file_path, output_diff, version_from, version_to): """Create concatenated diff for project file diffs between versions version_from and version_to. @@ -1401,11 +1420,11 @@ def download_files( """ Download project files at specified version. Get the latest if no version specified. - :param project_dir: project local directory or a full project name ("/") + :param project_dir: project local directory :type project_dir: String :param file_path: List of relative paths of files to download in the project directory :type file_path: List[String] - :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir (only valid when project_dir is an existing local checkout). + :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir. :type output_paths: List[String] :param version: optional version tag for downloaded file :type version: String diff --git a/mergin/client_pull.py b/mergin/client_pull.py index c4d1c7b..634ead0 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -22,7 +22,7 @@ import concurrent.futures -from .common import CHUNK_SIZE, ClientError, DeltaChangeType, InvalidProject, PullActionType +from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject from .utils import cleanup_tmp_dir, is_versioned_file, save_to_file @@ -82,8 +82,8 @@ def dump(self): class DownloadScratchContext: """ - Minimal stand-in for MerginProject, used by download_files_async() when downloading files - directly by project name ("/") without an existing local project checkout. + Minimal stand-in for MerginProject used when downloading a file directly by + project name ("/") without an existing local project checkout. Provides only what the shared download job code actually needs from MerginProject. """ @@ -793,6 +793,23 @@ def download_file_finalize(job): download_files_finalize(job) +def download_project_file_async(mc, project_path: str, file_path: str, output_file: str, version: str = None): + """ + Starts background download of a single project file at specified version, fetched directly + from the server without needing an existing local project checkout. + Returns handle to the pending download. + + :param project_path: full project name ("/") + :param output_file: destination path for the downloaded file + """ + if not output_file: + raise ClientError("output_file must be provided when downloading a file without a local project checkout") + + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + mp = DownloadScratchContext(mc, tmp_dir.name) + return _download_files_async(mc, mp, project_path, [file_path], [output_file], version, tmp_dir) + + def download_diffs_async(mc, project_directory, file_path, versions): """ Starts background download project file diffs for specified versions. @@ -916,35 +933,29 @@ def download_diffs_finalize(job: PullJob) -> List[str]: def download_files_async( - mc, project_dir: str, file_paths: typing.List[str], output_paths: typing.List[str], version: str + mc, project_dir: str, file_paths: typing.List[str], output_paths: typing.List[str] = None, version: str = None ): """ Starts background download project files at specified version. Returns handle to the pending download. - `project_dir` can either be an existing local project directory (previously fetched with - download_project()), or a full project name ("/") to download files - directly from the server without needing a local checkout. In the latter case, `output_paths` - must be provided explicitly, as there is no project directory to place files into by default. + `project_dir` must be an existing local project directory. """ - # temporary directory to stage downloaded chunks in + mp = MerginProject(project_dir) + project_path = mp.project_full_name() tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + return _download_files_async(mc, mp, project_path, file_paths, output_paths, version, tmp_dir) - mp: Union[MerginProject, "DownloadScratchContext"] - try: - mp = MerginProject(project_dir) - project_path = mp.project_full_name() - except InvalidProject: - # project_dir is not an existing local checkout - treat it as a full project name - # ("/") and download straight from the server instead - if output_paths is None: - cleanup_tmp_dir(mc, tmp_dir) - raise ClientError( - "output_paths must be provided when downloading files without an existing local project checkout" - ) - project_path = project_dir - mp = DownloadScratchContext(mc, tmp_dir.name) +def _download_files_async( + mc, + mp: Union[MerginProject, "DownloadScratchContext"], + project_path: str, + file_paths: typing.List[str], + output_paths: typing.List[str], + version: str, + tmp_dir: tempfile.TemporaryDirectory, +): ver_info = f"at version {version}" if version is not None else "at latest version" mp.log.info(f"Getting [{', '.join(file_paths)}] {ver_info}") latest_proj_info = mc.project_info(project_path) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index d84c7b8..cae4d2a 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1371,23 +1371,23 @@ def test_download_file_without_checkout(mc): f_downloaded = os.path.join(download_dir, f_updated) expected_content = "inserted_1_A.gpkg" - mc.download_file(project, f_updated, f_downloaded, version="v2") + mc.download_project_file(project, f_updated, f_downloaded, version="v2") expected = os.path.join(TEST_DATA_DIR, expected_content) assert check_gpkg_same_content(MerginProject(project_dir), f_downloaded, expected) assert not os.path.exists(os.path.join(download_dir, ".mergin")) - # output_paths must be provided explicitly when there is no local checkout - with pytest.raises(ClientError, match="output_paths must be provided"): - mc.download_files(project, [f_updated]) + # output_file must be provided explicitly when there is no local checkout + with pytest.raises(ClientError, match="output_file must be provided"): + mc.download_project_file(project, f_updated, None) # non-existent file in an existing project - same error as with a local checkout with pytest.raises(ClientError, match=r"No \[does_not_exist\.gpkg\] exists at version v2"): - mc.download_file(project, "does_not_exist.gpkg", f_downloaded, version="v2") + mc.download_project_file(project, "does_not_exist.gpkg", f_downloaded, version="v2") # non-existent / inaccessible project should fail clearly too nonexistent_project = create_project_path("this_project_does_not_exist", mc) with pytest.raises(ClientError): - mc.download_file(nonexistent_project, f_updated, f_downloaded) + mc.download_project_file(nonexistent_project, f_updated, f_downloaded) def test_download_diffs(mc): From 36718492854e3b620b368c6e9c86d448aa4dc329 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 10 Sep 2026 14:14:09 +0200 Subject: [PATCH 5/5] Use separate hidden file for file filter settings --- mergin/client_pull.py | 12 ++++-------- mergin/client_push.py | 9 ++------- mergin/merginproject.py | 17 ++++++++++++++--- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 3366bb4..971188d 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -284,8 +284,9 @@ def download_project_async(mc, project_path, directory, project_version=None, in # keep only the files matching the filter (if any) project_info["files"] = filter_files(project_info["files"], include=include, exclude=exclude) + # persisted once since it must never change again for this checkout if include or exclude: - project_info["file_filter"] = {"include": include, "exclude": exclude} + mp.write_file_filter({"include": include, "exclude": exclude}) # prepare download update_tasks = [] # stuff to do at the end of download @@ -762,13 +763,8 @@ def pull_project_finalize(job: PullJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise ClientError("Failed to apply pull actions: " + str(e)) - file_filter = job.mp.file_filter() - job.project_info["files"] = filter_files(job.project_info["files"], **file_filter) - # keep the sparse checkout: re-apply the filter this project was downloaded with, - # since job.project_info is a fresh, unfiltered response from the server - # and update_metadata() replaces the whole metadata dict rather than merging into it - if file_filter["include"] or file_filter["exclude"]: - job.project_info["file_filter"] = file_filter + # keep only in-scope files in the metadata we're about to persist + job.project_info["files"] = filter_files(job.project_info["files"], **job.mp.file_filter()) job.mp.update_metadata(job.project_info) diff --git a/mergin/client_push.py b/mergin/client_push.py index 1c9d3d7..155d270 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -458,13 +458,8 @@ def push_project_finalize(job: UploadJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise err - # keep the sparse checkout: re-apply the filter this project was - # downloaded with, since job.server_resp is a fresh, unfiltered response from the server - # and update_metadata() replaces the whole metadata dict rather than merging into it - file_filter = job.mp.file_filter() - job.server_resp["files"] = filter_files(job.server_resp["files"], **file_filter) - if file_filter["include"] or file_filter["exclude"]: - job.server_resp["file_filter"] = file_filter + # keep only in-scope files in the metadata we're about to persist + job.server_resp["files"] = filter_files(job.server_resp["files"], **job.mp.file_filter()) job.mp.update_metadata(job.server_resp) try: diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 3058a45..a8da37f 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -220,10 +220,21 @@ def files(self) -> list: def file_filter(self) -> dict: """ - Returns the include/exclude file filter this project was downloaded with, as a dict with "include" and "exclude" keys. + Returns the include/exclude file filter this project was downloaded with, as a dict + with "include" and "exclude" keys. Stored in its own file (.mergin/file_filter.json) """ - self._read_metadata() - return self._metadata.get("file_filter", {"include": None, "exclude": None}) + filter_file = self.fpath_meta("file_filter.json") + if not os.path.exists(filter_file): + return {"include": None, "exclude": None} + with open(filter_file, "r") as f: + return json.load(f) + + def write_file_filter(self, file_filter: dict) -> None: + """ + Persists the include/exclude file filter this project was downloaded with. + """ + with open(self.fpath_meta("file_filter.json"), "w") as f: + json.dump(file_filter, f, indent=2) @property def metadata(self) -> dict: