From 02a68ca8cfdfd0d6e3eede4ad54787cd0eb6454a Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 18 Nov 2026 16:55:25 +0100 Subject: [PATCH 1/4] Let large Harmony pushes from FwHeadless finish Lexbox copied pushed commits into the temp table 10 rows per COPY, so a ~45k-commit push spent its whole 10 s attempt on round trips and rolled back every retry. Use linq2db's default batch size (10,000): rows still stream one at a time, so memory and Npgsql timeouts don't change. FwHeadless's Harmony sync now uses its own HTTP client with a 30 s attempt and 90 s total timeout; other Lexbox calls keep 10 s / 30 s. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/FwHeadlessKernel.cs | 22 +++++++- .../FwHeadless/Services/CrdtSyncService.cs | 2 +- .../LexBoxApi/Services/CrdtCommitService.cs | 3 +- .../FwHeadless/LexboxHttpClientTests.cs | 52 +++++++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 backend/Testing/FwHeadless/LexboxHttpClientTests.cs diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index e96157c7da..89a389404d 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -14,6 +14,7 @@ namespace FwHeadless; public static class FwHeadlessKernel { public const string LexboxHttpClientName = "LexboxHttpClient"; + public const string LexboxSyncHttpClientName = "LexboxSyncHttpClient"; public static IServiceCollection AddFwHeadless(this IServiceCollection services) { services @@ -51,11 +52,28 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddTransient(); - services.AddHttpClient(LexboxHttpClientName, + services.AddLexboxHttpClient(LexboxHttpClientName); + // Lexbox stores a push in one transaction, so a project's whole unpushed history must fit in one attempt +#pragma warning disable EXTEXP0001 + services.AddLexboxHttpClient(LexboxSyncHttpClientName) + .RemoveAllResilienceHandlers() +#pragma warning restore EXTEXP0001 + .AddStandardResilienceHandler(options => + { + options.AttemptTimeout.Timeout = TimeSpan.FromSeconds(30); + options.TotalRequestTimeout.Timeout = TimeSpan.FromSeconds(90); + // must be at least twice the attempt timeout + options.CircuitBreaker.SamplingDuration = TimeSpan.FromSeconds(60); + }); + return services; + } + + private static IHttpClientBuilder AddLexboxHttpClient(this IServiceCollection services, string name) + { + return services.AddHttpClient(name, (provider, client) => { client.BaseAddress = new Uri(provider.GetRequiredService>().Value.LexboxUrl); }).AddHttpMessageHandler(); - return services; } } diff --git a/backend/FwHeadless/Services/CrdtSyncService.cs b/backend/FwHeadless/Services/CrdtSyncService.cs index 7fbaf3d87f..2223f63d0e 100644 --- a/backend/FwHeadless/Services/CrdtSyncService.cs +++ b/backend/FwHeadless/Services/CrdtSyncService.cs @@ -17,7 +17,7 @@ public virtual async Task SyncHarmonyProject() activity?.SetTag("app.project_id", currentProjectService.ProjectData.Id); var lexboxRemoteServer = await httpSyncService.CreateProjectSyncable( currentProjectService.ProjectData, - httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName) + httpClientFactory.CreateClient(FwHeadlessKernel.LexboxSyncHttpClientName) ); var syncResults = await dataModel.SyncWith(lexboxRemoteServer); if (!syncResults.IsSynced) throw new InvalidOperationException("Sync failed"); diff --git a/backend/LexBoxApi/Services/CrdtCommitService.cs b/backend/LexBoxApi/Services/CrdtCommitService.cs index a7f287be06..fd0fd33af3 100644 --- a/backend/LexBoxApi/Services/CrdtCommitService.cs +++ b/backend/LexBoxApi/Services/CrdtCommitService.cs @@ -21,7 +21,8 @@ public async Task AddCommits(Guid projectId, IAsyncEnumerable //A projection lambda here would let linq2db v6 wrap our Sql.Expr<...>::jsonb cast in the //EF value-converter (JsonSerializer.Serialize) and fail SQL translation. var stampedCommits = commits.Select(c => { c.ProjectId = projectId; return c; }); - await tmpTable.BulkCopyAsync(new BulkCopyOptions{BulkCopyType = BulkCopyType.ProviderSpecific, MaxBatchSize = 10}, stampedCommits, token); + //rows stream into COPY either way; a small MaxBatchSize only adds a server round trip per batch + await tmpTable.BulkCopyAsync(new BulkCopyOptions{BulkCopyType = BulkCopyType.ProviderSpecific}, stampedCommits, token); var commitsTable = linqToDbContext.GetTable(); var inserted = await commitsTable diff --git a/backend/Testing/FwHeadless/LexboxHttpClientTests.cs b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs new file mode 100644 index 0000000000..92da36b433 --- /dev/null +++ b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs @@ -0,0 +1,52 @@ +using FwHeadless; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Http.Resilience; +using Microsoft.Extensions.Options; +using WebServiceDefaults; + +namespace Testing.FwHeadless; + +public class LexboxHttpClientTests +{ + private static ServiceProvider BuildServices() + { + var builder = Host.CreateApplicationBuilder(); + builder.AddServiceDefaults("test"); + builder.Services.AddFwHeadless(); + return builder.Services.BuildServiceProvider(); + } + + private static int CountResilienceHandlers(HttpMessageHandler handler) + { + var count = 0; + while (handler is DelegatingHandler delegating) + { + if (delegating is ResilienceHandler) count++; + handler = delegating.InnerHandler!; + } + return count; + } + + [Fact] + public void SyncClient_ReplacesTheDefaultResilienceHandler() + { + using var services = BuildServices(); + var handler = services.GetRequiredService().CreateHandler(FwHeadlessKernel.LexboxSyncHttpClientName); + + CountResilienceHandlers(handler).Should().Be(1); + var options = services.GetRequiredService>() + .Get($"{FwHeadlessKernel.LexboxSyncHttpClientName}-standard"); + options.AttemptTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(30)); + options.TotalRequestTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(90)); + } + + [Fact] + public void DefaultClient_KeepsTheStandardTimeouts() + { + using var services = BuildServices(); + var options = services.GetRequiredService>() + .Get($"{FwHeadlessKernel.LexboxHttpClientName}-standard"); + options.AttemptTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(10)); + } +} From 1e16fb41e9f248f3326bc40fd55e6c459de08c47 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 18 Nov 2026 17:07:10 +0100 Subject: [PATCH 2/4] Run auth on every retry of the Lexbox sync client Replacing the default resilience handler put the new one after the auth handler, so retries reused the first attempt's cookie. Register auth after it, matching the default client, and test the handler order of both clients. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/FwHeadlessKernel.cs | 13 ++++--- .../FwHeadless/LexboxHttpClientTests.cs | 35 ++++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index 89a389404d..0073762cc7 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -52,11 +52,13 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddTransient(); - services.AddLexboxHttpClient(LexboxHttpClientName); - // Lexbox stores a push in one transaction, so a project's whole unpushed history must fit in one attempt + services.AddLexboxHttpClient(LexboxHttpClientName) + .AddHttpMessageHandler(); + // Lexbox stores a push in one transaction, so a project's whole unpushed history must fit in one attempt. + // The resilience handler goes before auth, like the default one, so every retry re-runs auth. + var syncClient = services.AddLexboxHttpClient(LexboxSyncHttpClientName); #pragma warning disable EXTEXP0001 - services.AddLexboxHttpClient(LexboxSyncHttpClientName) - .RemoveAllResilienceHandlers() + syncClient.RemoveAllResilienceHandlers() #pragma warning restore EXTEXP0001 .AddStandardResilienceHandler(options => { @@ -65,6 +67,7 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) // must be at least twice the attempt timeout options.CircuitBreaker.SamplingDuration = TimeSpan.FromSeconds(60); }); + syncClient.AddHttpMessageHandler(); return services; } @@ -74,6 +77,6 @@ private static IHttpClientBuilder AddLexboxHttpClient(this IServiceCollection se (provider, client) => { client.BaseAddress = new Uri(provider.GetRequiredService>().Value.LexboxUrl); - }).AddHttpMessageHandler(); + }); } } diff --git a/backend/Testing/FwHeadless/LexboxHttpClientTests.cs b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs index 92da36b433..24a9e1ed76 100644 --- a/backend/Testing/FwHeadless/LexboxHttpClientTests.cs +++ b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs @@ -17,26 +17,37 @@ private static ServiceProvider BuildServices() return builder.Services.BuildServiceProvider(); } - private static int CountResilienceHandlers(HttpMessageHandler handler) + private static List HandlerChain(ServiceProvider services, string clientName) { - var count = 0; + var handler = services.GetRequiredService().CreateHandler(clientName); + var chain = new List(); while (handler is DelegatingHandler delegating) { - if (delegating is ResilienceHandler) count++; + if (delegating is ResilienceHandler or HttpClientAuthHandler) chain.Add(delegating.GetType()); handler = delegating.InnerHandler!; } - return count; + return chain; } - [Fact] - public void SyncClient_ReplacesTheDefaultResilienceHandler() + private static HttpStandardResilienceOptions StandardOptions(ServiceProvider services, string clientName) + { + return services.GetRequiredService>().Get($"{clientName}-standard"); + } + + [Theory] + [InlineData(FwHeadlessKernel.LexboxHttpClientName)] + [InlineData(FwHeadlessKernel.LexboxSyncHttpClientName)] + public void EachRetryRunsAuth(string clientName) { using var services = BuildServices(); - var handler = services.GetRequiredService().CreateHandler(FwHeadlessKernel.LexboxSyncHttpClientName); + HandlerChain(services, clientName).Should().Equal(typeof(ResilienceHandler), typeof(HttpClientAuthHandler)); + } - CountResilienceHandlers(handler).Should().Be(1); - var options = services.GetRequiredService>() - .Get($"{FwHeadlessKernel.LexboxSyncHttpClientName}-standard"); + [Fact] + public void SyncClient_UsesLongerTimeouts() + { + using var services = BuildServices(); + var options = StandardOptions(services, FwHeadlessKernel.LexboxSyncHttpClientName); options.AttemptTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(30)); options.TotalRequestTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(90)); } @@ -45,8 +56,8 @@ public void SyncClient_ReplacesTheDefaultResilienceHandler() public void DefaultClient_KeepsTheStandardTimeouts() { using var services = BuildServices(); - var options = services.GetRequiredService>() - .Get($"{FwHeadlessKernel.LexboxHttpClientName}-standard"); + var options = StandardOptions(services, FwHeadlessKernel.LexboxHttpClientName); options.AttemptTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(10)); + options.TotalRequestTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(30)); } } From 975731fbf6f3e47f7847e22cc4622b132d5482e4 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Thu, 19 Nov 2026 07:25:59 +0000 Subject: [PATCH 3/4] Add the auth handler inside AddLexboxHttpClient and drop the timeout tests Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/FwHeadlessKernel.cs | 39 +++++++++++-------- .../FwHeadless/LexboxHttpClientTests.cs | 23 ----------- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index 0073762cc7..b4edf107e6 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -5,6 +5,7 @@ using FwLiteProjectSync; using LcmCrdt; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Http.Resilience; using Microsoft.Extensions.Options; using MiniLcm.Project; using MiniLcmWeb; @@ -52,31 +53,35 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddTransient(); - services.AddLexboxHttpClient(LexboxHttpClientName) - .AddHttpMessageHandler(); + services.AddLexboxHttpClient(LexboxHttpClientName); // Lexbox stores a push in one transaction, so a project's whole unpushed history must fit in one attempt. - // The resilience handler goes before auth, like the default one, so every retry re-runs auth. - var syncClient = services.AddLexboxHttpClient(LexboxSyncHttpClientName); -#pragma warning disable EXTEXP0001 - syncClient.RemoveAllResilienceHandlers() -#pragma warning restore EXTEXP0001 - .AddStandardResilienceHandler(options => - { - options.AttemptTimeout.Timeout = TimeSpan.FromSeconds(30); - options.TotalRequestTimeout.Timeout = TimeSpan.FromSeconds(90); - // must be at least twice the attempt timeout - options.CircuitBreaker.SamplingDuration = TimeSpan.FromSeconds(60); - }); - syncClient.AddHttpMessageHandler(); + services.AddLexboxHttpClient(LexboxSyncHttpClientName, options => + { + options.AttemptTimeout.Timeout = TimeSpan.FromSeconds(30); + options.TotalRequestTimeout.Timeout = TimeSpan.FromSeconds(90); + // must be at least twice the attempt timeout + options.CircuitBreaker.SamplingDuration = TimeSpan.FromSeconds(60); + }); return services; } - private static IHttpClientBuilder AddLexboxHttpClient(this IServiceCollection services, string name) + private static IHttpClientBuilder AddLexboxHttpClient(this IServiceCollection services, string name, + Action? configureResilience = null) { - return services.AddHttpClient(name, + var builder = services.AddHttpClient(name, (provider, client) => { client.BaseAddress = new Uri(provider.GetRequiredService>().Value.LexboxUrl); }); + if (configureResilience is not null) + { +#pragma warning disable EXTEXP0001 + builder.RemoveAllResilienceHandlers() +#pragma warning restore EXTEXP0001 + .AddStandardResilienceHandler(configureResilience); + } + // Auth goes after the resilience handler (the default one is added first), so every retry re-runs auth. + builder.AddHttpMessageHandler(); + return builder; } } diff --git a/backend/Testing/FwHeadless/LexboxHttpClientTests.cs b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs index 24a9e1ed76..5093ffaae0 100644 --- a/backend/Testing/FwHeadless/LexboxHttpClientTests.cs +++ b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Http.Resilience; -using Microsoft.Extensions.Options; using WebServiceDefaults; namespace Testing.FwHeadless; @@ -29,11 +28,6 @@ private static List HandlerChain(ServiceProvider services, string clientNa return chain; } - private static HttpStandardResilienceOptions StandardOptions(ServiceProvider services, string clientName) - { - return services.GetRequiredService>().Get($"{clientName}-standard"); - } - [Theory] [InlineData(FwHeadlessKernel.LexboxHttpClientName)] [InlineData(FwHeadlessKernel.LexboxSyncHttpClientName)] @@ -43,21 +37,4 @@ public void EachRetryRunsAuth(string clientName) HandlerChain(services, clientName).Should().Equal(typeof(ResilienceHandler), typeof(HttpClientAuthHandler)); } - [Fact] - public void SyncClient_UsesLongerTimeouts() - { - using var services = BuildServices(); - var options = StandardOptions(services, FwHeadlessKernel.LexboxSyncHttpClientName); - options.AttemptTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(30)); - options.TotalRequestTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(90)); - } - - [Fact] - public void DefaultClient_KeepsTheStandardTimeouts() - { - using var services = BuildServices(); - var options = StandardOptions(services, FwHeadlessKernel.LexboxHttpClientName); - options.AttemptTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(10)); - options.TotalRequestTimeout.Timeout.Should().Be(TimeSpan.FromSeconds(30)); - } } From a7479419c77572a4c365de8f1850f80bdb613b26 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Thu, 19 Nov 2026 07:26:08 +0000 Subject: [PATCH 4/4] Remove stray blank line Co-Authored-By: Claude Opus 5.5 --- backend/Testing/FwHeadless/LexboxHttpClientTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/Testing/FwHeadless/LexboxHttpClientTests.cs b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs index 5093ffaae0..9aaa5d9384 100644 --- a/backend/Testing/FwHeadless/LexboxHttpClientTests.cs +++ b/backend/Testing/FwHeadless/LexboxHttpClientTests.cs @@ -36,5 +36,4 @@ public void EachRetryRunsAuth(string clientName) using var services = BuildServices(); HandlerChain(services, clientName).Should().Equal(typeof(ResilienceHandler), typeof(HttpClientAuthHandler)); } - }