Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions src/SIL.Harmony.Tests/Syncable/SyncableTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,14 @@ public async Task AddRangeFromSync_IsIdempotent(ISyncableTestBackend backend)
{
await using var context = await backend.CreateAsync();
var commit = SetWordCommit("entity1", context.ClientId);
await context.Syncable.AddRangeFromSync([commit]);
var firstAdd = await context.Syncable.AddRangeFromSync([commit]);
var stateBefore = await context.Syncable.GetSyncState();

await context.Syncable.AddRangeFromSync([commit]);
var secondAdd = await context.Syncable.AddRangeFromSync([commit]);
var stateAfter = await context.Syncable.GetSyncState();

firstAdd.Should().ContainSingle().Which.Id.Should().Be(commit.Id);
secondAdd.Should().BeEmpty();
stateAfter.Should().BeEquivalentTo(stateBefore);
var changes = await context.Syncable.GetChanges(new SyncState([], []));
changes.MissingFromClient.Should().HaveCount(1);
Expand All @@ -210,6 +212,29 @@ public async Task SyncWith_ReturnsMissingCommits(ISyncableTestBackend localBacke
syncResults.MissingFromRemote.Should().ContainSingle(c => c.Id == commit.Id);
}

[Theory]
[MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))]
public async Task SyncWith_ReturnsOnlyCommitsTheReceiverDidNotHave(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend)
{
await using var local = await localBackend.CreateAsync();
await using var remote = await remoteBackend.CreateAsync();
var date = new DateTimeOffset(2000, 1, 1, 0, 0, 0, TimeSpan.Zero);
var shared = SetWordCommit("shared", local.ClientId, date);
await local.Syncable.AddRangeFromSync([shared]);
await local.Syncable.SyncWith(remote.Syncable);

//same head and count for the client but different commits, so each side resends its whole history for it
var onlyLocal = SetWordCommit("local", local.ClientId, date.AddDays(-1));
await local.Syncable.AddRangeFromSync([onlyLocal]);
var onlyRemote = SetWordCommit("remote", local.ClientId, date.AddDays(-2));
await remote.Syncable.AddRangeFromSync([onlyRemote]);

var results = await local.Syncable.SyncWith(remote.Syncable);

results.MissingFromLocal.Should().ContainSingle().Which.Id.Should().Be(onlyRemote.Id);
results.MissingFromRemote.Should().ContainSingle().Which.Id.Should().Be(onlyLocal.Id);
}

[Theory]
[MemberData(nameof(SyncableTestHelpers.BackendPairData), MemberType = typeof(SyncableTestHelpers))]
public async Task CanSyncSimpleChange(ISyncableTestBackend localBackend, ISyncableTestBackend remoteBackend)
Expand Down
10 changes: 5 additions & 5 deletions src/SIL.Harmony/DataModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -140,18 +140,17 @@ internal static ChangeEntity<IChange> ToChangeEntity(IChange change, int index,
};
}

async Task ISyncable.AddRangeFromSync(IEnumerable<Commit> commits)
async Task<Commit[]> ISyncable.AddRangeFromSync(IEnumerable<Commit> commitsEnumerable)
{
commits = commits.ToArray();
var commits = commitsEnumerable.ToArray();
try
{
await using var repo = await _crdtRepositoryFactory.CreateRepository();
using var locked = await repo.Lock();
repo.ClearChangeTracker();
_timeProvider.TakeLatestTime(commits.Select(c => c.HybridDateTime));
var (oldestChange, newCommits) = await repo.FilterExistingCommits(commits.ToArray());
//no changes added
if (oldestChange is null || newCommits is []) return;
var (oldestChange, newCommits) = await repo.FilterExistingCommits(commits);
if (oldestChange is null || newCommits is []) return [];
CrdtRepository.ReplayWindow replayWindow;
await using (var transaction = await repo.BeginTransactionAsync())
{
Expand All @@ -163,6 +162,7 @@ async Task ISyncable.AddRangeFromSync(IEnumerable<Commit> commits)
//a window resuming from nothing rebuilt every snapshot, the first sync into an empty database is the
//big one, since it pulls the whole project. Any later full rebuild is small, so compacting after it is cheap
if (replayWindow.ResumeFrom is null) await repo.CompactDatabase();
return newCommits;
}
catch (DbUpdateException e)
{
Expand Down
7 changes: 4 additions & 3 deletions src/SIL.Harmony/ISyncable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ namespace SIL.Harmony;

public interface ISyncable
{
Task AddRangeFromSync(IEnumerable<Commit> commits);
/// <returns>The commits this syncable didn't already have. One that can't tell returns all it was given.</returns>
Task<Commit[]> AddRangeFromSync(IEnumerable<Commit> commits);
Task<SyncState> GetSyncState();
Task<ChangesResult<Commit>> GetChanges(SyncState otherHeads);
Task<SyncResults> SyncWith(ISyncable remoteModel);
Expand All @@ -14,9 +15,9 @@ public class NullSyncable : ISyncable
{
public static readonly ISyncable Instance = new NullSyncable();

public Task AddRangeFromSync(IEnumerable<Commit> commits)
public Task<Commit[]> AddRangeFromSync(IEnumerable<Commit> commits)
{
return Task.CompletedTask;
return Task.FromResult<Commit[]>([]);
}

public Task<SyncState> GetSyncState()
Expand Down
12 changes: 8 additions & 4 deletions src/SIL.Harmony/JsonSyncable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ public JsonSyncable(DirectoryInfo dir, JsonSerializerOptions serializerOptions,
_syncService = syncService;
}

public async Task AddRangeFromSync(IEnumerable<Commit> commits)
public async Task<Commit[]> AddRangeFromSync(IEnumerable<Commit> commits)
{
var commitArray = commits.ToArray();
if (commitArray.Length == 0) return;
Comment thread
myieye marked this conversation as resolved.
Resolved
var groups = commits.GroupBy(c => c.ClientId).ToArray();
if (groups.Length == 0) return [];

var groups = commitArray.GroupBy(c => c.ClientId).ToArray();
var added = new ConcurrentBag<Commit>();
await Parallel.ForEachAsync(groups, async (group, ct) =>
{
using (await ClientLocks.GetOrAdd(group.Key, _ => new AsyncLock()).LockAsync())
Expand All @@ -45,9 +45,13 @@ await Parallel.ForEachAsync(groups, async (group, ct) =>

await using var stream = new StreamWriter(file.FullName, append: true);
foreach (var commit in newCommits)
{
WriteCommit(stream, commit);
added.Add(commit);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
}
});
return [.. added.DefaultOrder()];
}

public async Task<SyncState> GetSyncState()
Expand Down
19 changes: 10 additions & 9 deletions src/SIL.Harmony/SyncService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ public async Task<SyncResults> SyncWith(ISyncable localModel, ISyncable remoteMo
}
var pullTask = TryAddRange(localModel, missingFromLocal);
var pushTask = TryAddRange(remoteModel, missingFromRemote);
ThrowIfAnyErrors([await pullTask, await pushTask]);
return new SyncResults(missingFromLocal, missingFromRemote, true);
var (pulled, pullError) = await pullTask;
var (pushed, pushError) = await pushTask;
ThrowIfAnyErrors([pullError, pushError]);
return new SyncResults(pulled, pushed, true);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
public async Task SyncMany(ISyncable localModel, ISyncable[] remotes)
{
Expand All @@ -49,7 +51,7 @@ public async Task SyncMany(ISyncable localModel, ISyncable[] remotes)
missingFromLocal = Clone(missingFromLocal);
}
remoteSyncStates[i] = remoteSyncState;
errors.Add(await TryAddRange(localModel, missingFromLocal));
errors.Add((await TryAddRange(localModel, missingFromLocal)).Error);
}
// Now the localModel has the changes it could apply from all remotes, so all remotes will get the changes from the localModel as well as all other remotes
for (var i = 0; i < remotes.Length; i++)
Expand All @@ -62,23 +64,22 @@ public async Task SyncMany(ISyncable localModel, ISyncable[] remotes)
//cloning just to simulate the objects going over the wire
missingFromRemote = Clone(missingFromRemote);
}
errors.Add(await TryAddRange(remote, missingFromRemote));
errors.Add((await TryAddRange(remote, missingFromRemote)).Error);
}
ThrowIfAnyErrors(errors);
}

//don't fail fast: if neither direction lands, the next sync sends the same commits and fails the same way
private static async Task<Exception?> TryAddRange(ISyncable model, Commit[] commits)
private static async Task<(Commit[] Added, Exception? Error)> TryAddRange(ISyncable model, Commit[] commits)
{
if (commits.Length == 0) return null;
if (commits.Length == 0) return ([], null);
try
{
await model.AddRangeFromSync(commits);
return null;
return (await model.AddRangeFromSync(commits), null);
}
catch (Exception e)
{
return e;
return ([], e);
}
}

Expand Down
Loading