fix force update

deployment/community/.env.template

@@ -121,6 +121,8 @@ LOCAL_PROJECTS=/data
 
 #MAX_CHUNK_SIZE=10 * 1024 * 1024  # 10485760 in bytes
 
+#MAX_DIFFABLE_FORCE_UPDATE_SIZE=512 * 1024 * 1024  # 536870912 in bytes - max size of an uploaded full .gpkg for which server tries to construct a diff on force update, above this it falls back to a plain full-file force update
+
 # data download
 
 #MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10  # max total files size in bytes for archive download - 10 GB

deployment/enterprise/.env.template

@@ -118,6 +118,8 @@ LOCAL_PROJECTS=/data
 
 #MAX_CHUNK_SIZE=10 * 1024 * 1024  # 10485760 in bytes
 
+#MAX_DIFFABLE_FORCE_UPDATE_SIZE=512 * 1024 * 1024  # 536870912 in bytes - max size of an uploaded full .gpkg for which server tries to construct a diff on force update, above this it falls back to a plain full-file force update
+
 # data download
 
 #MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10  # max total files size in bytes for archive download

server/mergin/sync/config.py

@@ -88,3 +88,7 @@ class Configuration(object):
     )
     # max batch size for fetch projects in batch endpoint
     MAX_BATCH_SIZE = config("MAX_BATCH_SIZE", default=100, cast=int)
+    # max size (in bytes) of an uploaded full .gpkg file for which server will try to construct a diff
+    MAX_DIFFABLE_FORCE_UPDATE_SIZE = config(
+        "MAX_DIFFABLE_FORCE_UPDATE_SIZE", default=512 * 1024 * 1024, cast=int
+    )

server/mergin/sync/models.py

@@ -2163,7 +2163,11 @@ def process_chunks(
                             errors[f.path] = (
                                 f"{FileSyncErrorType.SYNC_ERROR.value}: project {self.project.workspace.name}/{self.project.name}, {result.value}"
                             )
-                    else:
+                    elif (
+                        expected_size
+                        <= current_app.config["MAX_DIFFABLE_FORCE_UPDATE_SIZE"]
+                    ):
+                        # gpkg small enough - try to construct diff server-side
                         diff_name = mergin_secure_filename(
                             f.path + "-diff-" + str(uuid.uuid4())
                         )
@@ -2188,6 +2192,12 @@ def process_chunks(
                             logging.warning(
                                 f"Geodiff: create changeset error {result.value}"
                             )
+                    else:
+                        # gpkg too large - skip diff construction and keep it as a plain force update
+                        logging.info(
+                            f"Skipping diff construction for {f.path} in project {project_path}: "
+                            f"file size {expected_size} exceeds MAX_DIFFABLE_FORCE_UPDATE_SIZE"
+                        )
         return file_changes, errors
 
 

server/mergin/tests/test_project_controller.py

@@ -1755,6 +1755,102 @@ def copy_file_failing_for_geodiff(src, dest):
     assert "diff" not in updated_file
 
 
+def test_push_force_update_size_limit(client):
+    """Server should only try to construct a diff for a force-updated (full gpkg,
+    no diff sent) upload when its size is within MAX_DIFFABLE_FORCE_UPDATE_SIZE;
+    above the limit it should skip diff construction and keep it as a plain
+    force update."""
+    working_dir = os.path.join(TMP_DIR, "test_push_force_update_size_limit")
+    # cleanup
+    if os.path.exists(working_dir):
+        shutil.rmtree(working_dir)
+
+    shutil.copytree(test_project_dir, working_dir)
+    # mimic base.gpkg was updated with inserted_1_A.gpkg (but no diff is created)
+    shutil.copy(
+        os.path.join(working_dir, "inserted_1_A.gpkg"),
+        os.path.join(working_dir, "base.gpkg"),
+    )
+    base_gpkg_size = os.path.getsize(os.path.join(working_dir, "base.gpkg"))
+    changes = {
+        "added": [],
+        "removed": [],
+        "updated": [
+            file_info(working_dir, "base.gpkg", chunk_size=CHUNK_SIZE),
+            file_info(working_dir, "test.txt", chunk_size=CHUNK_SIZE),
+        ],
+    }
+
+    # below limit -> diff is still constructed server-side
+    upload, upload_dir = create_transaction("mergin", changes)
+    upload_chunks(upload_dir, upload.changes, src_dir=working_dir)
+    with patch.dict(
+        client.application.config,
+        {"MAX_DIFFABLE_FORCE_UPDATE_SIZE": base_gpkg_size + 1},
+    ):
+        resp = client.post(f"/v1/project/push/finish/{upload.transaction_id}")
+    assert resp.status_code == 200
+    latest_version = upload.project.get_latest_version()
+    assert (
+        latest_version.changes.filter(
+            FileHistory.change == PushChangeType.UPDATE.value
+        ).count()
+        == 1
+    )
+    assert (
+        latest_version.changes.filter(
+            FileHistory.change == PushChangeType.UPDATE_DIFF.value
+        ).count()
+        == 1
+    )
+    file_meta = latest_version.changes.filter(
+        FileHistory.change == PushChangeType.UPDATE_DIFF.value
+    ).first()
+    assert file_meta.diff_file is not None
+    assert os.path.exists(
+        os.path.join(upload.project.storage.project_dir, file_meta.diff_file.location)
+    )
+
+    # above limit -> diff construction is skipped, plain force update
+    working_file = os.path.join(working_dir, "base.gpkg")
+    sql = "INSERT INTO simple (geometry, name) VALUES (GeomFromText('POINT(24.5, 38.2)', 4326), 'insert_test')"
+    execute_query(working_file, sql)
+    updated_gpkg_size = os.path.getsize(working_file)
+    changes["updated"] = [
+        file_info(working_dir, "base.gpkg", chunk_size=CHUNK_SIZE),
+        file_info(working_dir, "test.txt", chunk_size=CHUNK_SIZE),
+    ]
+    upload, upload_dir = create_transaction("mergin", changes, version=2)
+    upload_chunks(upload_dir, upload.changes, src_dir=working_dir)
+    with patch.dict(
+        client.application.config,
+        {"MAX_DIFFABLE_FORCE_UPDATE_SIZE": updated_gpkg_size - 1},
+    ):
+        resp = client.post(f"/v1/project/push/finish/{upload.transaction_id}")
+    assert resp.status_code == 200
+    latest_version = upload.project.get_latest_version()
+    assert (
+        latest_version.changes.filter(
+            FileHistory.change == PushChangeType.UPDATE.value
+        ).count()
+        == 2
+    )
+    assert not latest_version.changes.filter(
+        FileHistory.change == PushChangeType.UPDATE_DIFF.value
+    ).count()
+    assert all(
+        file_meta.diff_file is None
+        for file_meta in latest_version.changes.filter(
+            FileHistory.change == PushChangeType.UPDATE.value
+        ).all()
+    )
+    version_files = os.listdir(
+        os.path.join(upload.project.storage.project_dir, f"v{latest_version.name}")
+    )
+    diff_files = [f for f in version_files if re.findall("-diff-", f)]
+    assert not diff_files
+
+
 clone_project_data = [
     ({"project": " clone "}, "mergin", 200),  # clone own project
     (