From 798c4df25493cd2cd39850c251884285bf84b6cb Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 18 Nov 2026 15:15:36 +0100 Subject: [PATCH 1/5] Report only newly added commits in SyncResults AddRangeFromSync skips commits the receiver already has, but SyncWith returned everything the peer sent, so a recovery resend reported a client's whole history as new. ISyncable gains AddRangeFromSyncReturningNew (default: returns all commits, so existing implementations keep compiling); DataModel and JsonSyncable return what they actually stored. Fixes #174 Co-Authored-By: Claude Opus 5.5 --- .../Syncable/SyncRecoveryTests.cs | 22 +++++++++++++++++++ src/SIL.Harmony/DataModel.cs | 12 +++++----- src/SIL.Harmony/ISyncable.cs | 8 +++++++ src/SIL.Harmony/JsonSyncable.cs | 14 ++++++++---- src/SIL.Harmony/SyncService.cs | 19 ++++++++-------- 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs b/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs index 3896e6f..6780f0b 100644 --- a/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs +++ b/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs @@ -143,6 +143,28 @@ public async Task Sync_CanRecoverBothClientsHavingAsymmetricCountOfOutOfOrderCom await ShouldHaveCommit(remote, commit0.Id); } + [Theory] + [MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))] + public async Task Sync_RecoveryResend_ReportsOnlyCommitsNewToEachSide(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend) + { + await using var local = await localBackend.CreateAsync(); + await using var remote = await remoteBackend.CreateAsync(); + var shared = SetWordCommit("shared", local.ClientId, Date); + await local.Syncable.AddRangeFromSync([shared]); + await local.Syncable.SyncWith(remote.Syncable); + + //same head and count for the client but different commits, so each side resends its whole history for it + var onlyLocal = SetWordCommit("local", local.ClientId, Date.AddDays(-1)); + await local.Syncable.AddRangeFromSync([onlyLocal]); + var onlyRemote = SetWordCommit("remote", local.ClientId, Date.AddDays(-2)); + await remote.Syncable.AddRangeFromSync([onlyRemote]); + + var results = await local.Syncable.SyncWith(remote.Syncable); + + results.MissingFromLocal.Should().ContainSingle().Which.Id.Should().Be(onlyRemote.Id); + results.MissingFromRemote.Should().ContainSingle().Which.Id.Should().Be(onlyLocal.Id); + } + private static readonly DateTimeOffset Date = new(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); //behind has fewer and older commits from the shared client, so PlanFor withholds the create but still sends a third client's edit to it diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index fc6f495..190e60b 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -140,18 +140,19 @@ internal static ChangeEntity ToChangeEntity(IChange change, int index, }; } - async Task ISyncable.AddRangeFromSync(IEnumerable commits) + Task ISyncable.AddRangeFromSync(IEnumerable commits) => + ((ISyncable)this).AddRangeFromSyncReturningNew(commits.ToArray()); + + async Task ISyncable.AddRangeFromSyncReturningNew(Commit[] commits) { - commits = commits.ToArray(); try { await using var repo = await _crdtRepositoryFactory.CreateRepository(); using var locked = await repo.Lock(); repo.ClearChangeTracker(); _timeProvider.TakeLatestTime(commits.Select(c => c.HybridDateTime)); - var (oldestChange, newCommits) = await repo.FilterExistingCommits(commits.ToArray()); - //no changes added - if (oldestChange is null || newCommits is []) return; + var (oldestChange, newCommits) = await repo.FilterExistingCommits(commits); + if (oldestChange is null || newCommits is []) return []; CrdtRepository.ReplayWindow replayWindow; await using (var transaction = await repo.BeginTransactionAsync()) { @@ -163,6 +164,7 @@ async Task ISyncable.AddRangeFromSync(IEnumerable commits) //a window resuming from nothing rebuilt every snapshot, the first sync into an empty database is the //big one, since it pulls the whole project. Any later full rebuild is small, so compacting after it is cheap if (replayWindow.ResumeFrom is null) await repo.CompactDatabase(); + return newCommits; } catch (DbUpdateException e) { diff --git a/src/SIL.Harmony/ISyncable.cs b/src/SIL.Harmony/ISyncable.cs index 5e7d750..f65ee35 100644 --- a/src/SIL.Harmony/ISyncable.cs +++ b/src/SIL.Harmony/ISyncable.cs @@ -3,6 +3,14 @@ namespace SIL.Harmony; public interface ISyncable { Task AddRangeFromSync(IEnumerable commits); + + /// The commits this syncable didn't already have. The default can't tell, so it returns them all. + async Task AddRangeFromSyncReturningNew(Commit[] commits) + { + await AddRangeFromSync(commits); + return commits; + } + Task GetSyncState(); Task> GetChanges(SyncState otherHeads); Task SyncWith(ISyncable remoteModel); diff --git a/src/SIL.Harmony/JsonSyncable.cs b/src/SIL.Harmony/JsonSyncable.cs index 3815c20..2088d77 100644 --- a/src/SIL.Harmony/JsonSyncable.cs +++ b/src/SIL.Harmony/JsonSyncable.cs @@ -27,12 +27,14 @@ public JsonSyncable(DirectoryInfo dir, JsonSerializerOptions serializerOptions, _syncService = syncService; } - public async Task AddRangeFromSync(IEnumerable commits) + public Task AddRangeFromSync(IEnumerable commits) => AddRangeFromSyncReturningNew(commits.ToArray()); + + public async Task AddRangeFromSyncReturningNew(Commit[] commits) { - var commitArray = commits.ToArray(); - if (commitArray.Length == 0) return; + if (commits.Length == 0) return []; - var groups = commitArray.GroupBy(c => c.ClientId).ToArray(); + var added = new ConcurrentBag(); + var groups = commits.GroupBy(c => c.ClientId).ToArray(); await Parallel.ForEachAsync(groups, async (group, ct) => { using (await ClientLocks.GetOrAdd(group.Key, _ => new AsyncLock()).LockAsync()) @@ -45,9 +47,13 @@ await Parallel.ForEachAsync(groups, async (group, ct) => await using var stream = new StreamWriter(file.FullName, append: true); foreach (var commit in newCommits) + { WriteCommit(stream, commit); + added.Add(commit); + } } }); + return [.. added.DefaultOrder()]; } public async Task GetSyncState() diff --git a/src/SIL.Harmony/SyncService.cs b/src/SIL.Harmony/SyncService.cs index ad2d377..30217db 100644 --- a/src/SIL.Harmony/SyncService.cs +++ b/src/SIL.Harmony/SyncService.cs @@ -31,8 +31,10 @@ public async Task SyncWith(ISyncable localModel, ISyncable remoteMo } var pullTask = TryAddRange(localModel, missingFromLocal); var pushTask = TryAddRange(remoteModel, missingFromRemote); - ThrowIfAnyErrors([await pullTask, await pushTask]); - return new SyncResults(missingFromLocal, missingFromRemote, true); + var (pulled, pullError) = await pullTask; + var (pushed, pushError) = await pushTask; + ThrowIfAnyErrors([pullError, pushError]); + return new SyncResults(pulled, pushed, true); } public async Task SyncMany(ISyncable localModel, ISyncable[] remotes) { @@ -49,7 +51,7 @@ public async Task SyncMany(ISyncable localModel, ISyncable[] remotes) missingFromLocal = Clone(missingFromLocal); } remoteSyncStates[i] = remoteSyncState; - errors.Add(await TryAddRange(localModel, missingFromLocal)); + errors.Add((await TryAddRange(localModel, missingFromLocal)).Error); } // Now the localModel has the changes it could apply from all remotes, so all remotes will get the changes from the localModel as well as all other remotes for (var i = 0; i < remotes.Length; i++) @@ -62,23 +64,22 @@ public async Task SyncMany(ISyncable localModel, ISyncable[] remotes) //cloning just to simulate the objects going over the wire missingFromRemote = Clone(missingFromRemote); } - errors.Add(await TryAddRange(remote, missingFromRemote)); + errors.Add((await TryAddRange(remote, missingFromRemote)).Error); } ThrowIfAnyErrors(errors); } //don't fail fast: if neither direction lands, the next sync sends the same commits and fails the same way - private static async Task TryAddRange(ISyncable model, Commit[] commits) + private static async Task<(Commit[] Added, Exception? Error)> TryAddRange(ISyncable model, Commit[] commits) { - if (commits.Length == 0) return null; + if (commits.Length == 0) return ([], null); try { - await model.AddRangeFromSync(commits); - return null; + return (await model.AddRangeFromSyncReturningNew(commits), null); } catch (Exception e) { - return e; + return ([], e); } } From 6558ccbf3c757dd9ace77f5b861b18efcb07923d Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 18 Nov 2026 16:10:13 +0100 Subject: [PATCH 2/5] Return the added commits from AddRangeFromSync instead of adding a second method Breaking for ISyncable implementers; an implementer that can't tell returns everything it was given. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony/DataModel.cs | 6 ++---- src/SIL.Harmony/ISyncable.cs | 15 ++++----------- src/SIL.Harmony/JsonSyncable.cs | 6 +----- src/SIL.Harmony/SyncService.cs | 2 +- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 190e60b..4405584 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -140,11 +140,9 @@ internal static ChangeEntity ToChangeEntity(IChange change, int index, }; } - Task ISyncable.AddRangeFromSync(IEnumerable commits) => - ((ISyncable)this).AddRangeFromSyncReturningNew(commits.ToArray()); - - async Task ISyncable.AddRangeFromSyncReturningNew(Commit[] commits) + async Task ISyncable.AddRangeFromSync(IEnumerable commitsEnumerable) { + var commits = commitsEnumerable.ToArray(); try { await using var repo = await _crdtRepositoryFactory.CreateRepository(); diff --git a/src/SIL.Harmony/ISyncable.cs b/src/SIL.Harmony/ISyncable.cs index f65ee35..b45bf31 100644 --- a/src/SIL.Harmony/ISyncable.cs +++ b/src/SIL.Harmony/ISyncable.cs @@ -2,15 +2,8 @@ namespace SIL.Harmony; public interface ISyncable { - Task AddRangeFromSync(IEnumerable commits); - - /// The commits this syncable didn't already have. The default can't tell, so it returns them all. - async Task AddRangeFromSyncReturningNew(Commit[] commits) - { - await AddRangeFromSync(commits); - return commits; - } - + /// The commits this syncable didn't already have. One that can't tell returns all it was given. + Task AddRangeFromSync(IEnumerable commits); Task GetSyncState(); Task> GetChanges(SyncState otherHeads); Task SyncWith(ISyncable remoteModel); @@ -22,9 +15,9 @@ public class NullSyncable : ISyncable { public static readonly ISyncable Instance = new NullSyncable(); - public Task AddRangeFromSync(IEnumerable commits) + public Task AddRangeFromSync(IEnumerable commits) { - return Task.CompletedTask; + return Task.FromResult([]); } public Task GetSyncState() diff --git a/src/SIL.Harmony/JsonSyncable.cs b/src/SIL.Harmony/JsonSyncable.cs index 2088d77..7f6ef53 100644 --- a/src/SIL.Harmony/JsonSyncable.cs +++ b/src/SIL.Harmony/JsonSyncable.cs @@ -27,12 +27,8 @@ public JsonSyncable(DirectoryInfo dir, JsonSerializerOptions serializerOptions, _syncService = syncService; } - public Task AddRangeFromSync(IEnumerable commits) => AddRangeFromSyncReturningNew(commits.ToArray()); - - public async Task AddRangeFromSyncReturningNew(Commit[] commits) + public async Task AddRangeFromSync(IEnumerable commits) { - if (commits.Length == 0) return []; - var added = new ConcurrentBag(); var groups = commits.GroupBy(c => c.ClientId).ToArray(); await Parallel.ForEachAsync(groups, async (group, ct) => diff --git a/src/SIL.Harmony/SyncService.cs b/src/SIL.Harmony/SyncService.cs index 30217db..d64f04f 100644 --- a/src/SIL.Harmony/SyncService.cs +++ b/src/SIL.Harmony/SyncService.cs @@ -75,7 +75,7 @@ public async Task SyncMany(ISyncable localModel, ISyncable[] remotes) if (commits.Length == 0) return ([], null); try { - return (await model.AddRangeFromSyncReturningNew(commits), null); + return (await model.AddRangeFromSync(commits), null); } catch (Exception e) { From 8e8138cb54c48b9c0d6fcd589322c17afdca4aae Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 18 Nov 2026 16:25:24 +0100 Subject: [PATCH 3/5] Move the SyncWith reporting test to SyncableTests and check AddRangeFromSync's return Co-Authored-By: Claude Opus 5.5 --- .../Syncable/SyncRecoveryTests.cs | 22 ---------------- .../Syncable/SyncableTests.cs | 26 ++++++++++++++++++- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs b/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs index 6780f0b..3896e6f 100644 --- a/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs +++ b/src/SIL.Harmony.Tests/Syncable/SyncRecoveryTests.cs @@ -143,28 +143,6 @@ public async Task Sync_CanRecoverBothClientsHavingAsymmetricCountOfOutOfOrderCom await ShouldHaveCommit(remote, commit0.Id); } - [Theory] - [MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))] - public async Task Sync_RecoveryResend_ReportsOnlyCommitsNewToEachSide(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend) - { - await using var local = await localBackend.CreateAsync(); - await using var remote = await remoteBackend.CreateAsync(); - var shared = SetWordCommit("shared", local.ClientId, Date); - await local.Syncable.AddRangeFromSync([shared]); - await local.Syncable.SyncWith(remote.Syncable); - - //same head and count for the client but different commits, so each side resends its whole history for it - var onlyLocal = SetWordCommit("local", local.ClientId, Date.AddDays(-1)); - await local.Syncable.AddRangeFromSync([onlyLocal]); - var onlyRemote = SetWordCommit("remote", local.ClientId, Date.AddDays(-2)); - await remote.Syncable.AddRangeFromSync([onlyRemote]); - - var results = await local.Syncable.SyncWith(remote.Syncable); - - results.MissingFromLocal.Should().ContainSingle().Which.Id.Should().Be(onlyRemote.Id); - results.MissingFromRemote.Should().ContainSingle().Which.Id.Should().Be(onlyLocal.Id); - } - private static readonly DateTimeOffset Date = new(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); //behind has fewer and older commits from the shared client, so PlanFor withholds the create but still sends a third client's edit to it diff --git a/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs b/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs index 0c58bd5..d1958ab 100644 --- a/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs +++ b/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs @@ -189,9 +189,10 @@ public async Task AddRangeFromSync_IsIdempotent(ISyncableTestBackend backend) await context.Syncable.AddRangeFromSync([commit]); var stateBefore = await context.Syncable.GetSyncState(); - await context.Syncable.AddRangeFromSync([commit]); + var added = await context.Syncable.AddRangeFromSync([commit]); var stateAfter = await context.Syncable.GetSyncState(); + added.Should().BeEmpty(); stateAfter.Should().BeEquivalentTo(stateBefore); var changes = await context.Syncable.GetChanges(new SyncState([], [])); changes.MissingFromClient.Should().HaveCount(1); @@ -210,6 +211,29 @@ public async Task SyncWith_ReturnsMissingCommits(ISyncableTestBackend localBacke syncResults.MissingFromRemote.Should().ContainSingle(c => c.Id == commit.Id); } + [Theory] + [MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))] + public async Task SyncWith_ReturnsOnlyCommitsTheReceiverDidNotHave(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend) + { + await using var local = await localBackend.CreateAsync(); + await using var remote = await remoteBackend.CreateAsync(); + var date = new DateTimeOffset(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); + var shared = SetWordCommit("shared", local.ClientId, date); + await local.Syncable.AddRangeFromSync([shared]); + await local.Syncable.SyncWith(remote.Syncable); + + //same head and count for the client but different commits, so each side resends its whole history for it + var onlyLocal = SetWordCommit("local", local.ClientId, date.AddDays(-1)); + await local.Syncable.AddRangeFromSync([onlyLocal]); + var onlyRemote = SetWordCommit("remote", local.ClientId, date.AddDays(-2)); + await remote.Syncable.AddRangeFromSync([onlyRemote]); + + var results = await local.Syncable.SyncWith(remote.Syncable); + + results.MissingFromLocal.Should().ContainSingle().Which.Id.Should().Be(onlyRemote.Id); + results.MissingFromRemote.Should().ContainSingle().Which.Id.Should().Be(onlyLocal.Id); + } + [Theory] [MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))] public async Task CanSyncSimpleChange(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend) From 5089388fdb7b16953f4165cd897edd46aadb0eaa Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 18 Nov 2026 16:56:26 +0100 Subject: [PATCH 4/5] Check that AddRangeFromSync returns a newly added commit Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Tests/Syncable/SyncableTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs b/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs index d1958ab..fe89d1e 100644 --- a/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs +++ b/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs @@ -186,13 +186,14 @@ public async Task AddRangeFromSync_IsIdempotent(ISyncableTestBackend backend) { await using var context = await backend.CreateAsync(); var commit = SetWordCommit("entity1", context.ClientId); - await context.Syncable.AddRangeFromSync([commit]); + var firstAdd = await context.Syncable.AddRangeFromSync([commit]); var stateBefore = await context.Syncable.GetSyncState(); - var added = await context.Syncable.AddRangeFromSync([commit]); + var secondAdd = await context.Syncable.AddRangeFromSync([commit]); var stateAfter = await context.Syncable.GetSyncState(); - added.Should().BeEmpty(); + firstAdd.Should().ContainSingle().Which.Id.Should().Be(commit.Id); + secondAdd.Should().BeEmpty(); stateAfter.Should().BeEquivalentTo(stateBefore); var changes = await context.Syncable.GetChanges(new SyncState([], [])); changes.MissingFromClient.Should().HaveCount(1); From ce48d972af2cd449b18826ddead816d94cf3058c Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Thu, 19 Nov 2026 11:07:21 +0100 Subject: [PATCH 5/5] Restore JsonSyncable's early return for an empty batch Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony/JsonSyncable.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/SIL.Harmony/JsonSyncable.cs b/src/SIL.Harmony/JsonSyncable.cs index 7f6ef53..a234684 100644 --- a/src/SIL.Harmony/JsonSyncable.cs +++ b/src/SIL.Harmony/JsonSyncable.cs @@ -29,8 +29,10 @@ public JsonSyncable(DirectoryInfo dir, JsonSerializerOptions serializerOptions, public async Task AddRangeFromSync(IEnumerable commits) { - var added = new ConcurrentBag(); var groups = commits.GroupBy(c => c.ClientId).ToArray(); + if (groups.Length == 0) return []; + + var added = new ConcurrentBag(); await Parallel.ForEachAsync(groups, async (group, ct) => { using (await ClientLocks.GetOrAdd(group.Key, _ => new AsyncLock()).LockAsync())