diff --git a/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs b/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs index 0c58bd5..fe89d1e 100644 --- a/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs +++ b/src/SIL.Harmony.Tests/Syncable/SyncableTests.cs @@ -186,12 +186,14 @@ public async Task AddRangeFromSync_IsIdempotent(ISyncableTestBackend backend) { await using var context = await backend.CreateAsync(); var commit = SetWordCommit("entity1", context.ClientId); - await context.Syncable.AddRangeFromSync([commit]); + var firstAdd = await context.Syncable.AddRangeFromSync([commit]); var stateBefore = await context.Syncable.GetSyncState(); - await context.Syncable.AddRangeFromSync([commit]); + var secondAdd = await context.Syncable.AddRangeFromSync([commit]); var stateAfter = await context.Syncable.GetSyncState(); + firstAdd.Should().ContainSingle().Which.Id.Should().Be(commit.Id); + secondAdd.Should().BeEmpty(); stateAfter.Should().BeEquivalentTo(stateBefore); var changes = await context.Syncable.GetChanges(new SyncState([], [])); changes.MissingFromClient.Should().HaveCount(1); @@ -210,6 +212,29 @@ public async Task SyncWith_ReturnsMissingCommits(ISyncableTestBackend localBacke syncResults.MissingFromRemote.Should().ContainSingle(c => c.Id == commit.Id); } + [Theory] + [MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))] + public async Task SyncWith_ReturnsOnlyCommitsTheReceiverDidNotHave(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend) + { + await using var local = await localBackend.CreateAsync(); + await using var remote = await remoteBackend.CreateAsync(); + var date = new DateTimeOffset(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); + var shared = SetWordCommit("shared", local.ClientId, date); + await local.Syncable.AddRangeFromSync([shared]); + await local.Syncable.SyncWith(remote.Syncable); + + //same head and count for the client but different commits, so each side resends its whole history for it + var onlyLocal = SetWordCommit("local", local.ClientId, date.AddDays(-1)); + await local.Syncable.AddRangeFromSync([onlyLocal]); + var onlyRemote = SetWordCommit("remote", local.ClientId, date.AddDays(-2)); + await remote.Syncable.AddRangeFromSync([onlyRemote]); + + var results = await local.Syncable.SyncWith(remote.Syncable); + + results.MissingFromLocal.Should().ContainSingle().Which.Id.Should().Be(onlyRemote.Id); + results.MissingFromRemote.Should().ContainSingle().Which.Id.Should().Be(onlyLocal.Id); + } + [Theory] [MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))] public async Task CanSyncSimpleChange(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend) diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index fc6f495..4405584 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -140,18 +140,17 @@ internal static ChangeEntity ToChangeEntity(IChange change, int index, }; } - async Task ISyncable.AddRangeFromSync(IEnumerable commits) + async Task ISyncable.AddRangeFromSync(IEnumerable commitsEnumerable) { - commits = commits.ToArray(); + var commits = commitsEnumerable.ToArray(); try { await using var repo = await _crdtRepositoryFactory.CreateRepository(); using var locked = await repo.Lock(); repo.ClearChangeTracker(); _timeProvider.TakeLatestTime(commits.Select(c => c.HybridDateTime)); - var (oldestChange, newCommits) = await repo.FilterExistingCommits(commits.ToArray()); - //no changes added - if (oldestChange is null || newCommits is []) return; + var (oldestChange, newCommits) = await repo.FilterExistingCommits(commits); + if (oldestChange is null || newCommits is []) return []; CrdtRepository.ReplayWindow replayWindow; await using (var transaction = await repo.BeginTransactionAsync()) { @@ -163,6 +162,7 @@ async Task ISyncable.AddRangeFromSync(IEnumerable commits) //a window resuming from nothing rebuilt every snapshot, the first sync into an empty database is the //big one, since it pulls the whole project. Any later full rebuild is small, so compacting after it is cheap if (replayWindow.ResumeFrom is null) await repo.CompactDatabase(); + return newCommits; } catch (DbUpdateException e) { diff --git a/src/SIL.Harmony/ISyncable.cs b/src/SIL.Harmony/ISyncable.cs index 5e7d750..b45bf31 100644 --- a/src/SIL.Harmony/ISyncable.cs +++ b/src/SIL.Harmony/ISyncable.cs @@ -2,7 +2,8 @@ namespace SIL.Harmony; public interface ISyncable { - Task AddRangeFromSync(IEnumerable commits); + /// The commits this syncable didn't already have. One that can't tell returns all it was given. + Task AddRangeFromSync(IEnumerable commits); Task GetSyncState(); Task> GetChanges(SyncState otherHeads); Task SyncWith(ISyncable remoteModel); @@ -14,9 +15,9 @@ public class NullSyncable : ISyncable { public static readonly ISyncable Instance = new NullSyncable(); - public Task AddRangeFromSync(IEnumerable commits) + public Task AddRangeFromSync(IEnumerable commits) { - return Task.CompletedTask; + return Task.FromResult([]); } public Task GetSyncState() diff --git a/src/SIL.Harmony/JsonSyncable.cs b/src/SIL.Harmony/JsonSyncable.cs index 3815c20..a234684 100644 --- a/src/SIL.Harmony/JsonSyncable.cs +++ b/src/SIL.Harmony/JsonSyncable.cs @@ -27,12 +27,12 @@ public JsonSyncable(DirectoryInfo dir, JsonSerializerOptions serializerOptions, _syncService = syncService; } - public async Task AddRangeFromSync(IEnumerable commits) + public async Task AddRangeFromSync(IEnumerable commits) { - var commitArray = commits.ToArray(); - if (commitArray.Length == 0) return; + var groups = commits.GroupBy(c => c.ClientId).ToArray(); + if (groups.Length == 0) return []; - var groups = commitArray.GroupBy(c => c.ClientId).ToArray(); + var added = new ConcurrentBag(); await Parallel.ForEachAsync(groups, async (group, ct) => { using (await ClientLocks.GetOrAdd(group.Key, _ => new AsyncLock()).LockAsync()) @@ -45,9 +45,13 @@ await Parallel.ForEachAsync(groups, async (group, ct) => await using var stream = new StreamWriter(file.FullName, append: true); foreach (var commit in newCommits) + { WriteCommit(stream, commit); + added.Add(commit); + } } }); + return [.. added.DefaultOrder()]; } public async Task GetSyncState() diff --git a/src/SIL.Harmony/SyncService.cs b/src/SIL.Harmony/SyncService.cs index ad2d377..d64f04f 100644 --- a/src/SIL.Harmony/SyncService.cs +++ b/src/SIL.Harmony/SyncService.cs @@ -31,8 +31,10 @@ public async Task SyncWith(ISyncable localModel, ISyncable remoteMo } var pullTask = TryAddRange(localModel, missingFromLocal); var pushTask = TryAddRange(remoteModel, missingFromRemote); - ThrowIfAnyErrors([await pullTask, await pushTask]); - return new SyncResults(missingFromLocal, missingFromRemote, true); + var (pulled, pullError) = await pullTask; + var (pushed, pushError) = await pushTask; + ThrowIfAnyErrors([pullError, pushError]); + return new SyncResults(pulled, pushed, true); } public async Task SyncMany(ISyncable localModel, ISyncable[] remotes) { @@ -49,7 +51,7 @@ public async Task SyncMany(ISyncable localModel, ISyncable[] remotes) missingFromLocal = Clone(missingFromLocal); } remoteSyncStates[i] = remoteSyncState; - errors.Add(await TryAddRange(localModel, missingFromLocal)); + errors.Add((await TryAddRange(localModel, missingFromLocal)).Error); } // Now the localModel has the changes it could apply from all remotes, so all remotes will get the changes from the localModel as well as all other remotes for (var i = 0; i < remotes.Length; i++) @@ -62,23 +64,22 @@ public async Task SyncMany(ISyncable localModel, ISyncable[] remotes) //cloning just to simulate the objects going over the wire missingFromRemote = Clone(missingFromRemote); } - errors.Add(await TryAddRange(remote, missingFromRemote)); + errors.Add((await TryAddRange(remote, missingFromRemote)).Error); } ThrowIfAnyErrors(errors); } //don't fail fast: if neither direction lands, the next sync sends the same commits and fails the same way - private static async Task TryAddRange(ISyncable model, Commit[] commits) + private static async Task<(Commit[] Added, Exception? Error)> TryAddRange(ISyncable model, Commit[] commits) { - if (commits.Length == 0) return null; + if (commits.Length == 0) return ([], null); try { - await model.AddRangeFromSync(commits); - return null; + return (await model.AddRangeFromSync(commits), null); } catch (Exception e) { - return e; + return ([], e); } }