From 5c1844cc8734bd497a4eb7dc041ae7a95c9cf6f8 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Fri, 18 Sep 2026 08:34:18 +0200 Subject: [PATCH] Limit server-side diff construction for forced update Try to create a diff from full gpkg file on client's behalf only for certain file sizes. For large files do force update automatically. --- deployment/community/.env.template | 2 + deployment/enterprise/.env.template | 2 + server/mergin/sync/config.py | 4 + server/mergin/sync/models.py | 12 ++- .../mergin/tests/test_project_controller.py | 96 +++++++++++++++++++ 5 files changed, 115 insertions(+), 1 deletion(-) diff --git a/deployment/community/.env.template b/deployment/community/.env.template index cfb526ac..e786e52f 100644 --- a/deployment/community/.env.template +++ b/deployment/community/.env.template @@ -121,6 +121,8 @@ LOCAL_PROJECTS=/data #MAX_CHUNK_SIZE=10 * 1024 * 1024 # 10485760 in bytes +#MAX_DIFFABLE_FORCE_UPDATE_SIZE=512 * 1024 * 1024 # 536870912 in bytes - max size of an uploaded full .gpkg for which server tries to construct a diff on force update, above this it falls back to a plain full-file force update + # data download #MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10 # max total files size in bytes for archive download - 10 GB diff --git a/deployment/enterprise/.env.template b/deployment/enterprise/.env.template index ebdb8716..a11aa575 100644 --- a/deployment/enterprise/.env.template +++ b/deployment/enterprise/.env.template @@ -118,6 +118,8 @@ LOCAL_PROJECTS=/data #MAX_CHUNK_SIZE=10 * 1024 * 1024 # 10485760 in bytes +#MAX_DIFFABLE_FORCE_UPDATE_SIZE=512 * 1024 * 1024 # 536870912 in bytes - max size of an uploaded full .gpkg for which server tries to construct a diff on force update, above this it falls back to a plain full-file force update + # data download #MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10 # max total files size in bytes for archive download diff --git a/server/mergin/sync/config.py b/server/mergin/sync/config.py index a5c8167a..9b5648fa 100644 --- a/server/mergin/sync/config.py +++ b/server/mergin/sync/config.py @@ -88,3 +88,7 @@ class Configuration(object): ) # max batch size for fetch projects in batch endpoint MAX_BATCH_SIZE = config("MAX_BATCH_SIZE", default=100, cast=int) + # max size (in bytes) of an uploaded full .gpkg file for which server will try to construct a diff + MAX_DIFFABLE_FORCE_UPDATE_SIZE = config( + "MAX_DIFFABLE_FORCE_UPDATE_SIZE", default=512 * 1024 * 1024, cast=int + ) diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 3817acd6..d8bb55fc 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -2163,7 +2163,11 @@ def process_chunks( errors[f.path] = ( f"{FileSyncErrorType.SYNC_ERROR.value}: project {self.project.workspace.name}/{self.project.name}, {result.value}" ) - else: + elif ( + expected_size + <= current_app.config["MAX_DIFFABLE_FORCE_UPDATE_SIZE"] + ): + # gpkg small enough - try to construct diff server-side diff_name = mergin_secure_filename( f.path + "-diff-" + str(uuid.uuid4()) ) @@ -2188,6 +2192,12 @@ def process_chunks( logging.warning( f"Geodiff: create changeset error {result.value}" ) + else: + # gpkg too large - skip diff construction and keep it as a plain force update + logging.info( + f"Skipping diff construction for {f.path} in project {project_path}: " + f"file size {expected_size} exceeds MAX_DIFFABLE_FORCE_UPDATE_SIZE" + ) return file_changes, errors diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 1a0c76aa..fb8657f5 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -1755,6 +1755,102 @@ def copy_file_failing_for_geodiff(src, dest): assert "diff" not in updated_file +def test_push_force_update_size_limit(client): + """Server should only try to construct a diff for a force-updated (full gpkg, + no diff sent) upload when its size is within MAX_DIFFABLE_FORCE_UPDATE_SIZE; + above the limit it should skip diff construction and keep it as a plain + force update.""" + working_dir = os.path.join(TMP_DIR, "test_push_force_update_size_limit") + # cleanup + if os.path.exists(working_dir): + shutil.rmtree(working_dir) + + shutil.copytree(test_project_dir, working_dir) + # mimic base.gpkg was updated with inserted_1_A.gpkg (but no diff is created) + shutil.copy( + os.path.join(working_dir, "inserted_1_A.gpkg"), + os.path.join(working_dir, "base.gpkg"), + ) + base_gpkg_size = os.path.getsize(os.path.join(working_dir, "base.gpkg")) + changes = { + "added": [], + "removed": [], + "updated": [ + file_info(working_dir, "base.gpkg", chunk_size=CHUNK_SIZE), + file_info(working_dir, "test.txt", chunk_size=CHUNK_SIZE), + ], + } + + # below limit -> diff is still constructed server-side + upload, upload_dir = create_transaction("mergin", changes) + upload_chunks(upload_dir, upload.changes, src_dir=working_dir) + with patch.dict( + client.application.config, + {"MAX_DIFFABLE_FORCE_UPDATE_SIZE": base_gpkg_size + 1}, + ): + resp = client.post(f"/v1/project/push/finish/{upload.transaction_id}") + assert resp.status_code == 200 + latest_version = upload.project.get_latest_version() + assert ( + latest_version.changes.filter( + FileHistory.change == PushChangeType.UPDATE.value + ).count() + == 1 + ) + assert ( + latest_version.changes.filter( + FileHistory.change == PushChangeType.UPDATE_DIFF.value + ).count() + == 1 + ) + file_meta = latest_version.changes.filter( + FileHistory.change == PushChangeType.UPDATE_DIFF.value + ).first() + assert file_meta.diff_file is not None + assert os.path.exists( + os.path.join(upload.project.storage.project_dir, file_meta.diff_file.location) + ) + + # above limit -> diff construction is skipped, plain force update + working_file = os.path.join(working_dir, "base.gpkg") + sql = "INSERT INTO simple (geometry, name) VALUES (GeomFromText('POINT(24.5, 38.2)', 4326), 'insert_test')" + execute_query(working_file, sql) + updated_gpkg_size = os.path.getsize(working_file) + changes["updated"] = [ + file_info(working_dir, "base.gpkg", chunk_size=CHUNK_SIZE), + file_info(working_dir, "test.txt", chunk_size=CHUNK_SIZE), + ] + upload, upload_dir = create_transaction("mergin", changes, version=2) + upload_chunks(upload_dir, upload.changes, src_dir=working_dir) + with patch.dict( + client.application.config, + {"MAX_DIFFABLE_FORCE_UPDATE_SIZE": updated_gpkg_size - 1}, + ): + resp = client.post(f"/v1/project/push/finish/{upload.transaction_id}") + assert resp.status_code == 200 + latest_version = upload.project.get_latest_version() + assert ( + latest_version.changes.filter( + FileHistory.change == PushChangeType.UPDATE.value + ).count() + == 2 + ) + assert not latest_version.changes.filter( + FileHistory.change == PushChangeType.UPDATE_DIFF.value + ).count() + assert all( + file_meta.diff_file is None + for file_meta in latest_version.changes.filter( + FileHistory.change == PushChangeType.UPDATE.value + ).all() + ) + version_files = os.listdir( + os.path.join(upload.project.storage.project_dir, f"v{latest_version.name}") + ) + diff_files = [f for f in version_files if re.findall("-diff-", f)] + assert not diff_files + + clone_project_data = [ ({"project": " clone "}, "mergin", 200), # clone own project (