From 86fb8e0ec1f9aa0efc01996ce099da62562abb15 Mon Sep 17 00:00:00 2001 From: Quinn Klassen Date: Fri, 13 Nov 2026 18:19:33 -0700 Subject: [PATCH 1/2] Prioritize sticky workflow polls during autoscaling Balance sticky and normal admission using backlog, the sticky scaling\ntarget, and slot capacity instead of requiring both pools to have polls.\nWake admission waiters on changes and add regression coverage. --- CHANGELOG.md | 2 + .../temporal/internal/worker/AsyncPoller.java | 113 +++--------- .../worker/AsyncWorkflowPollTask.java | 39 ++++ .../worker/WorkflowPollerBalancer.java | 95 ++++++++++ .../internal/worker/WorkflowWorker.java | 12 +- .../tuning/PollerBehaviorAutoscaling.java | 4 + .../AsyncWorkflowPollerBalancingTest.java | 147 ++++++++++++++++ .../worker/WorkflowPollerBalancerTest.java | 166 ++++++++++++++++++ 8 files changed, 488 insertions(+), 90 deletions(-) create mode 100644 temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java create mode 100644 temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java create mode 100644 temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 7203bb04e1..7093202b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,8 @@ to docs, or any other relevant information. specific to the Temporal CLI dev server implementation and may no longer be valid if that implementation changes. ### Fixed +- Autoscaling workflow pollers now prioritize meaningful sticky backlog up to the sticky pool's + target and use slot capacity to balance sticky and non-sticky admission, matching Core's policy. - Skipped `Workflow.mutableSideEffect` calls made with the new recovery flag enabled no longer leave invalid marker commands after cache eviction or a worker restart. The flag remains disabled by default in this release to preserve replay compatibility during SDK rollout. diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java index 13f25e6c7a..cf79f70208 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java @@ -11,11 +11,7 @@ import io.temporal.worker.tuning.SlotSupplierFuture; import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.IntSupplier; -import javax.annotation.concurrent.ThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,15 +23,13 @@ final class AsyncPoller extends BasePoller { private static final Logger log = LoggerFactory.getLogger(AsyncPoller.class); private final TrackingSlotSupplier slotSupplier; private final SlotReservationData slotReservationData; - private final List> asyncTaskPollers; + private final List> asyncTaskPollers; private final PollerOptions pollerOptions; private final PollerBehaviorAutoscaling pollerBehavior; private final Scope workerMetricsScope; private Throttler pollRateThrottler; private final Thread.UncaughtExceptionHandler uncaughtExceptionHandler = new PollerUncaughtExceptionHandler(); - private final PollQueueBalancer pollerBalancer = - new PollQueueBalancer(); // Used to balance the number of slots across pollers AsyncPoller( TrackingSlotSupplier slotSupplier, @@ -58,7 +52,7 @@ final class AsyncPoller extends BasePoller { AsyncPoller( TrackingSlotSupplier slotSupplier, SlotReservationData slotReservationData, - List> asyncTaskPollers, + List> asyncTaskPollers, ShutdownableTaskExecutor taskExecutor, PollerOptions pollerOptions, NamespaceCapabilities namespaceCapabilities, @@ -120,11 +114,12 @@ public boolean start() { asyncTaskPoller.getLabel(), newTarget); pollerSemaphore.setMaxPermits(newTarget); + asyncTaskPoller.onPollerTargetChanged(newTarget); }); + asyncTaskPoller.onPollerTargetChanged(pollerBehavior.getInitialConcurrentTaskPollers()); asyncTaskPoller.setPollerTargetSupplier(pollScaleReportHandle::getTargetPollerCount); PollQueueTask pollQueue = new PollQueueTask(asyncTaskPoller, pollerSemaphore, pollScaleReportHandle); - pollerBalancer.addPoller(asyncTaskPoller.getLabel()); exec.execute(pollQueue); exec.scheduleAtFixedRate(pollScaleReportHandle, 0, 100, TimeUnit.MILLISECONDS); } @@ -172,6 +167,23 @@ default String getLabel() { default void setPollerTargetSupplier(IntSupplier pollerTargetSupplier) { // no-op } + + /** Waits for admission before reserving an execution slot. */ + default void awaitPollAdmission() throws InterruptedException { + // no-op + } + + default void onPollStarted() { + // no-op + } + + default void onPollFinished() { + // no-op + } + + default void onPollerTargetChanged(int target) { + // no-op + } } class PollQueueTask implements Runnable { @@ -229,7 +241,7 @@ public void run() { continue; } - pollerBalancer.balance(asyncTaskPoller.getLabel()); + asyncTaskPoller.awaitPollAdmission(); if (shouldTerminate()) { continue; } @@ -257,12 +269,12 @@ public void run() { CompletableFuture pollRequest = asyncTaskPoller.poll(permit); // Mark that we have made a poll request pollRequestMade = true; - pollerBalancer.startPoll(asyncTaskPoller.getLabel()); + asyncTaskPoller.onPollStarted(); pollRequest .handle( (task, e) -> { - pollerBalancer.endPoll(asyncTaskPoller.getLabel()); + asyncTaskPoller.onPollFinished(); if (e instanceof CompletionException) { e = e.getCause(); } @@ -310,7 +322,6 @@ public void run() { } if (shouldTerminate()) { - pollerBalancer.removePoller(asyncTaskPoller.getLabel()); abort = true; log.debug( "Poll loop is terminated: {} - {}", @@ -321,80 +332,4 @@ public void run() { } } } - - /** - * PollQueueBalancer is used to ensure that at least one poll request is running for each task - * type. This is necessary to avoid one task type from consuming all the slots and starving other - * pollers. - */ - @ThreadSafe - class PollQueueBalancer { - Map taskCounts = new HashMap<>(); - private final Lock balancerLock = new ReentrantLock(); - private final Condition balancerCondition = balancerLock.newCondition(); - - void startPoll(String pollerName) { - balancerLock.lock(); - Integer currentPolls = taskCounts.compute(pollerName, (k, v) -> v + 1); - if (currentPolls == 1) { - balancerCondition.signalAll(); - } - balancerLock.unlock(); - } - - void endPoll(String pollerName) { - balancerLock.lock(); - if (!taskCounts.containsKey(pollerName)) { - balancerLock.unlock(); - return; - } - Integer currentPolls = taskCounts.compute(pollerName, (k, v) -> v - 1); - if (currentPolls == 0) { - balancerCondition.signalAll(); - } - balancerLock.unlock(); - } - - void addPoller(String pollerName) { - balancerLock.lock(); - taskCounts.put(pollerName, 0); - balancerCondition.signalAll(); - balancerLock.unlock(); - } - - void removePoller(String pollerName) { - balancerLock.lock(); - taskCounts.remove(pollerName); - balancerCondition.signalAll(); - balancerLock.unlock(); - } - - /** Ensure that at least one poller is running for each task. */ - void balance(String p) throws InterruptedException { - while (!shouldTerminate()) { - balancerLock.lock(); - try { - // If this poller has no tasks then we can unblock immediately - if (taskCounts.get(p) == 0) { - return; - } - // Check if all tasks have at least one poll request - boolean allOtherTasksHavePolls = true; - for (String task : taskCounts.keySet()) { - if (!Objects.equals(task, p) && taskCounts.get(task) == 0) { - allOtherTasksHavePolls = false; - break; - } - } - if (!allOtherTasksHavePolls) { - balancerCondition.await(); - } else { - return; - } - } finally { - balancerLock.unlock(); - } - } - } - } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java index 085cb931f3..7a152eca3b 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java @@ -41,6 +41,7 @@ public class AsyncWorkflowPollTask private final Context.CancellableContext grpcContext = Context.ROOT.withCancellation(); private final AtomicBoolean shutdown = new AtomicBoolean(false); private final PollerTracker pollerTracker; + @Nullable private WorkflowPollerBalancer pollerBalancer; @Override public String toString() { @@ -120,6 +121,38 @@ public AsyncWorkflowPollTask( } } + void setPollerBalancer(WorkflowPollerBalancer pollerBalancer) { + this.pollerBalancer = pollerBalancer; + } + + @Override + public void awaitPollAdmission() throws InterruptedException { + if (pollerBalancer != null) { + pollerBalancer.awaitAdmission(stickyPoller); + } + } + + @Override + public void onPollStarted() { + if (pollerBalancer != null) { + pollerBalancer.pollStarted(stickyPoller); + } + } + + @Override + public void onPollFinished() { + if (pollerBalancer != null) { + pollerBalancer.pollFinished(stickyPoller); + } + } + + @Override + public void onPollerTargetChanged(int target) { + if (stickyPoller && pollerBalancer != null) { + pollerBalancer.setStickyTarget(target); + } + } + /** * Supplies the poller pool's current scaling target. Installed by AsyncPoller, which owns the * scale report handle; null until then, and for a pool that is not autoscaling. @@ -180,6 +213,9 @@ public CompletableFuture poll(SlotPermit permit) return response .thenApply( r -> { + if (stickyPoller && pollerBalancer != null && r != null) { + pollerBalancer.recordStickyBacklog(r.getBacklogCountHint()); + } if (r == null || r.getTaskToken().isEmpty()) { pollerMetricScope .counter(MetricsType.WORKFLOW_TASK_QUEUE_POLL_EMPTY_COUNTER) @@ -215,6 +251,9 @@ public void disableNormalPoll() { throw new IllegalStateException("Cannot disable normal poll for sticky poller"); } shutdown.set(true); + if (pollerBalancer != null) { + pollerBalancer.disableNormalPoll(); + } } @Override diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java new file mode 100644 index 0000000000..28f6767116 --- /dev/null +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java @@ -0,0 +1,95 @@ +package io.temporal.internal.worker; + +import java.util.Optional; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Coordinates autoscaling sticky and non-sticky poll admission before slot reservation. + * + *

Counts only outstanding polls, not executing tasks or pending reservations. Like Core's + * balancer, this policy does not guarantee that both queues always have an outstanding poll. + */ +@ThreadSafe +final class WorkflowPollerBalancer { + private final int maxSlots; + private int stickyPolls; + private int normalPolls; + private int stickyTarget; + private long stickyBacklog; + private boolean normalPollingDisabled; + + WorkflowPollerBalancer(Optional maxSlots, int initialStickyTarget) { + this.maxSlots = maxSlots.orElse(Integer.MAX_VALUE); + this.stickyTarget = initialStickyTarget; + } + + synchronized void awaitAdmission(boolean sticky) throws InterruptedException { + while (!canPoll(sticky)) { + wait(); + } + } + + private boolean canPoll(boolean sticky) { + boolean preferSticky = + stickyBacklog > 1 && stickyBacklog > stickyPolls && stickyPolls < stickyTarget; + if (sticky) { + if (stickyPolls == 0) { + return true; + } + if (stickyPolls >= stickyTarget) { + return false; + } + if (!normalPollingDisabled && normalPolls == 0 && stickyPolls + 1 >= maxSlots) { + return false; + } + if (preferSticky) { + return true; + } + } else { + // Let a disabled normal poller reach poll(), which aborts without issuing an RPC. + if (normalPollingDisabled || normalPolls == 0) { + return true; + } + if (stickyPolls == 0 && normalPolls + 1 >= maxSlots) { + return false; + } + if (preferSticky) { + return false; + } + } + return (long) stickyPolls + normalPolls < maxSlots; + } + + synchronized void pollStarted(boolean sticky) { + if (sticky) { + stickyPolls++; + } else { + normalPolls++; + } + notifyAll(); + } + + synchronized void pollFinished(boolean sticky) { + if (sticky) { + stickyPolls--; + } else { + normalPolls--; + } + notifyAll(); + } + + synchronized void recordStickyBacklog(long backlog) { + stickyBacklog = backlog; + notifyAll(); + } + + synchronized void setStickyTarget(int target) { + stickyTarget = target; + notifyAll(); + } + + synchronized void disableNormalPoll() { + normalPollingDisabled = true; + notifyAll(); + } +} diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java index 26c52ef27f..86105fb522 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java @@ -145,7 +145,7 @@ public boolean start() { boolean useAsyncPoller = pollerOptions.getPollerBehavior() instanceof PollerBehaviorAutoscaling; if (useAsyncPoller) { - List> pollers; + List pollers; if (stickyTaskQueueName != null) { AsyncWorkflowPollTask normalPoller = new AsyncWorkflowPollTask( @@ -195,6 +195,16 @@ public boolean start() { pollerTracker, workerControlTaskQueue())); } + if (stickyTaskQueueName != null) { + WorkflowPollerBalancer balancer = + new WorkflowPollerBalancer( + slotSupplier.maximumSlots(), + ((PollerBehaviorAutoscaling) pollerOptions.getPollerBehavior()) + .getInitialConcurrentTaskPollers()); + for (AsyncWorkflowPollTask task : pollers) { + task.setPollerBalancer(balancer); + } + } poller = new AsyncPoller<>( slotSupplier, diff --git a/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java b/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java index 7687c39350..f8e6df990f 100644 --- a/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java +++ b/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java @@ -9,6 +9,10 @@ * *

If the server does not support autoscaling, then the number of pollers will stay at the * initial number of pollers. + * + *

Sticky and non-sticky workflow queues scale independently while sharing execution slots. + * Admission prioritizes a meaningful sticky backlog while the sticky pool is below its target, + * without blocking the first poll of either queue type. */ public final class PollerBehaviorAutoscaling implements PollerBehavior { private final int minConcurrentTaskPollers; diff --git a/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java b/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java new file mode 100644 index 0000000000..c1922a7985 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java @@ -0,0 +1,147 @@ +package io.temporal.internal.worker; + +import static io.temporal.testUtils.Eventually.assertEventually; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; +import com.uber.m3.tally.NoopScope; +import io.grpc.Status; +import io.temporal.api.workflowservice.v1.GetSystemInfoResponse; +import io.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse; +import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.worker.tuning.*; +import java.time.Duration; +import java.util.Arrays; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +public class AsyncWorkflowPollerBalancingTest { + @Test(timeout = 10000) + public void onlyStickyResponsesUpdateBacklogIncludingEmptyResponses() throws Exception { + WorkflowServiceStubs service = mockService(); + when(service.futureStub().pollWorkflowTaskQueue(any())) + .thenReturn( + Futures.immediateFuture( + PollWorkflowTaskQueueResponse.newBuilder().setBacklogCountHint(10).build()), + Futures.immediateFuture( + PollWorkflowTaskQueueResponse.newBuilder().setBacklogCountHint(10).build()), + Futures.immediateFuture(PollWorkflowTaskQueueResponse.getDefaultInstance())); + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(new FixedSizeSlotSupplier<>(10), new NoopScope()); + WorkflowPollerBalancer balancer = spy(new WorkflowPollerBalancer(Optional.of(10), 3)); + AsyncWorkflowPollTask sticky = pollTask(service, slots, true, balancer); + AsyncWorkflowPollTask normal = pollTask(service, slots, false, balancer); + + normal.onPollerTargetChanged(1); + verify(balancer, never()).setStickyTarget(anyInt()); + sticky.onPollerTargetChanged(2); + verify(balancer).setStickyTarget(2); + + assertNull(normal.poll(new SlotPermit()).get(5, TimeUnit.SECONDS)); + verify(balancer, never()).recordStickyBacklog(anyLong()); + assertNull(sticky.poll(new SlotPermit()).get(5, TimeUnit.SECONDS)); + verify(balancer).recordStickyBacklog(10); + assertNull(sticky.poll(new SlotPermit()).get(5, TimeUnit.SECONDS)); + verify(balancer).recordStickyBacklog(0); + } + + @Test(timeout = 10000) + public void stickyPoolDoesNotReserveAnExtraSlotAtItsTarget() throws Exception { + WorkflowServiceStubs service = mockService(); + SettableFuture response = SettableFuture.create(); + when(service.futureStub().pollWorkflowTaskQueue(any())).thenReturn(response); + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(new FixedSizeSlotSupplier<>(10), new NoopScope()); + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.of(10), 1); + AtomicReference stickyThread = new AtomicReference<>(); + AtomicReference normalThread = new AtomicReference<>(); + AsyncWorkflowPollTask sticky = spy(pollTask(service, slots, true, balancer)); + AsyncWorkflowPollTask normal = spy(pollTask(service, slots, false, balancer)); + capturePollThread(sticky, stickyThread); + capturePollThread(normal, normalThread); + ShutdownableTaskExecutor executor = mock(ShutdownableTaskExecutor.class); + AsyncPoller poller = + new AsyncPoller<>( + slots, + new SlotReservationData("queue", "identity", ""), + Arrays.asList(sticky, normal), + executor, + PollerOptions.newBuilder() + .setPollThreadNamePrefix("balancing-test") + .setPollerBehavior(new PollerBehaviorAutoscaling(1, 3, 1)) + .build(), + new NamespaceCapabilities(), + new NoopScope()); + poller.start(); + try { + assertEventually( + Duration.ofSeconds(5), + () -> { + assertNotNull(stickyThread.get()); + assertNotNull(normalThread.get()); + assertEquals(Thread.State.WAITING, stickyThread.get().getState()); + assertEquals(Thread.State.WAITING, normalThread.get().getState()); + // One slot per outstanding poll, plus a normal slot waiting on poller concurrency. + // Sticky must wait for admission BEFORE reserving a second slot. + assertEquals(3, slots.getIssuedSlots()); + }); + verify(sticky, times(1)).onPollStarted(); + verify(normal, times(1)).onPollStarted(); + verify(service.futureStub(), times(2)).pollWorkflowTaskQueue(any()); + } finally { + poller.shutdown(new ShutdownManager(), true).get(5, TimeUnit.SECONDS); + response.setException(Status.CANCELLED.asRuntimeException()); + poller.awaitTermination(5, TimeUnit.SECONDS); + } + assertEventually(Duration.ofSeconds(5), () -> assertEquals(0, slots.getIssuedSlots())); + } + + private static void capturePollThread(AsyncWorkflowPollTask task, AtomicReference thread) + throws Exception { + doAnswer( + invocation -> { + thread.set(Thread.currentThread()); + return invocation.callRealMethod(); + }) + .when(task) + .awaitPollAdmission(); + } + + private static WorkflowServiceStubs mockService() { + WorkflowServiceStubs service = mock(WorkflowServiceStubs.class); + WorkflowServiceGrpc.WorkflowServiceFutureStub stub = + mock(WorkflowServiceGrpc.WorkflowServiceFutureStub.class); + when(service.futureStub()).thenReturn(stub); + when(stub.withOption(any(), any())).thenReturn(stub); + return service; + } + + private static AsyncWorkflowPollTask pollTask( + WorkflowServiceStubs service, + TrackingSlotSupplier slots, + boolean sticky, + WorkflowPollerBalancer balancer) { + AsyncWorkflowPollTask task = + new AsyncWorkflowPollTask( + service, + "default", + "queue", + sticky ? "sticky-queue" : null, + "identity", + "instance-key", + new WorkerVersioningOptions("", false, null), + slots, + new NoopScope(), + GetSystemInfoResponse.Capabilities::getDefaultInstance, + new PollerTracker(), + null); + task.setPollerBalancer(balancer); + return task; + } +} diff --git a/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java b/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java new file mode 100644 index 0000000000..b8e989fa67 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java @@ -0,0 +1,166 @@ +package io.temporal.internal.worker; + +import static io.temporal.testUtils.Eventually.assertEventually; +import static org.junit.Assert.*; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +public class WorkflowPollerBalancerTest { + @Test(timeout = 10000) + public void stickyPreferenceRequiresMeaningfulBacklog() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 3, 1, 1); + for (long backlog : new long[] {0, 1}) { + balancer.recordStickyBacklog(backlog); + balancer.awaitAdmission(false); + balancer.awaitAdmission(true); + } + balancer.recordStickyBacklog(10); + balancer.awaitAdmission(true); + assertBlocked(balancer, false); + + // A backlog no larger than the outstanding sticky polls is not meaningful. + balancer.pollStarted(true); + balancer.recordStickyBacklog(2); + balancer.awaitAdmission(false); + } + + @Test(timeout = 10000) + public void reachingStickyTargetReleasesPreference() throws Exception { + WorkflowPollerBalancer balancer = balancer(5, 3, 2, 1); + balancer.recordStickyBacklog(10); + assertUnblockedBy(balancer, false, () -> balancer.pollStarted(true)); + assertBlocked(balancer, true); + } + + @Test(timeout = 10000) + public void targetDecreaseWakesNormalWaiter() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 3, 2, 1); + balancer.recordStickyBacklog(10); + assertUnblockedBy(balancer, false, () -> balancer.setStickyTarget(2)); + assertBlocked(balancer, true); + } + + @Test(timeout = 10000) + public void targetIncreaseWakesStickyWaiter() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 1, 1, 1); + assertUnblockedBy(balancer, true, () -> balancer.setStickyTarget(2)); + } + + @Test(timeout = 10000) + public void backlogChangeWakesNormalWaiter() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 3, 1, 1); + balancer.recordStickyBacklog(10); + assertUnblockedBy(balancer, false, () -> balancer.recordStickyBacklog(1)); + } + + @Test(timeout = 10000) + public void pollCompletionWakesWaiterAtCapacity() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 4, 2, 2); + assertUnblockedBy(balancer, false, () -> balancer.pollFinished(true)); + balancer.pollStarted(true); + assertUnblockedBy(balancer, true, () -> balancer.pollFinished(false)); + } + + @Test(timeout = 10000) + public void firstPollOfEachTypeCanAttemptAdmission() throws Exception { + for (int capacity : new int[] {1, 2, 4}) { + WorkflowPollerBalancer balancer = balancer(capacity, 3, 0, 1); + balancer.awaitAdmission(true); + balancer = balancer(capacity, 3, 1, 0); + balancer.recordStickyBacklog(10); + balancer.awaitAdmission(false); + } + } + + @Test(timeout = 10000) + public void reserveRoomForOtherQueuesFirstPoll() throws Exception { + for (int capacity : new int[] {1, 2, 4}) { + assertBlocked(balancer(capacity, 10, Math.max(1, capacity - 1), 0), true); + assertBlocked(balancer(capacity, 10, 0, Math.max(1, capacity - 1)), false); + } + } + + @Test(timeout = 10000) + public void missingOtherPollDoesNotBlockSpareCapacity() throws Exception { + balancer(4, 3, 1, 0).awaitAdmission(true); + balancer(4, 3, 0, 1).awaitAdmission(false); + } + + @Test(timeout = 10000) + public void stickyTargetPreventsAdmissionEvenWithSpareSlots() throws Exception { + assertBlocked(balancer(10, 2, 2, 1), true); + assertBlocked(balancer(10, 1, 2, 1), true); + } + + @Test(timeout = 10000) + public void unknownCapacityStillRespectsBacklogAndTarget() throws Exception { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 2); + balancer.pollStarted(true); + balancer.pollStarted(false); + balancer.recordStickyBacklog(10); + assertBlocked(balancer, false); + balancer.pollStarted(true); + balancer.awaitAdmission(false); + assertBlocked(balancer, true); + } + + @Test(timeout = 10000) + public void disablingNormalPollingReleasesCapacityForStickyDrain() throws Exception { + WorkflowPollerBalancer balancer = balancer(2, 3, 1, 0); + assertUnblockedBy(balancer, true, balancer::disableNormalPoll); + } + + private static WorkflowPollerBalancer balancer( + int slots, int target, int stickyPolls, int normalPolls) { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.of(slots), target); + for (int i = 0; i < stickyPolls; i++) { + balancer.pollStarted(true); + } + for (int i = 0; i < normalPolls; i++) { + balancer.pollStarted(false); + } + return balancer; + } + + private static void assertBlocked(WorkflowPollerBalancer balancer, boolean sticky) + throws Exception { + assertWaiter(balancer, sticky, null); + } + + private static void assertUnblockedBy( + WorkflowPollerBalancer balancer, boolean sticky, Runnable change) throws Exception { + assertWaiter(balancer, sticky, change); + } + + private static void assertWaiter(WorkflowPollerBalancer balancer, boolean sticky, Runnable change) + throws Exception { + FutureTask admission = + new FutureTask<>( + () -> { + balancer.awaitAdmission(sticky); + return null; + }); + Thread thread = new Thread(admission, "balancer-test"); + thread.start(); + try { + assertEventually( + Duration.ofSeconds(5), + () -> { + assertFalse("Admission should be blocked", admission.isDone()); + assertEquals(Thread.State.WAITING, thread.getState()); + }); + if (change != null) { + change.run(); + admission.get(5, TimeUnit.SECONDS); + } + } finally { + thread.interrupt(); + thread.join(5000); + assertFalse("Admission waiter should terminate on interrupt", thread.isAlive()); + } + } +} From 7be3028ac09ce1431fee590972e97040f121863a Mon Sep 17 00:00:00 2001 From: Quinn Klassen Date: Wed, 18 Nov 2026 14:22:36 -0700 Subject: [PATCH 2/2] Preserve first-poll access without a slot maximum Wait for the other workflow queue's first poll when its slot capacity is unknown, rather than treating the limit as effectively unbounded. Cover both startup orders with an unadvertised two-slot supplier and add the incident recovery regression. Document the fallback explicitly. --- CHANGELOG.md | 4 +- .../worker/WorkflowPollerBalancer.java | 18 +- .../tuning/PollerBehaviorAutoscaling.java | 3 +- .../AsyncWorkflowPollerBalancingTest.java | 204 ++++++++++++++++++ .../worker/WorkflowPollerBalancerTest.java | 44 ++++ 5 files changed, 266 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7093202b7f..2cea9392ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,7 +112,9 @@ to docs, or any other relevant information. ### Fixed - Autoscaling workflow pollers now prioritize meaningful sticky backlog up to the sticky pool's - target and use slot capacity to balance sticky and non-sticky admission, matching Core's policy. + target and use advertised slot capacity to balance sticky and non-sticky admission. Without an + advertised maximum, additional polls of one queue type wait until the other has an active poll. + Unlike Core, Java does not use workflow cache size as a fallback slot bound. - Skipped `Workflow.mutableSideEffect` calls made with the new recovery flag enabled no longer leave invalid marker commands after cache eviction or a worker restart. The flag remains disabled by default in this release to preserve replay compatibility during SDK rollout. diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java index 28f6767116..a56b14e5a8 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java @@ -6,12 +6,16 @@ /** * Coordinates autoscaling sticky and non-sticky poll admission before slot reservation. * + *

Without an advertised slot maximum, additional polls of one type wait until the other type has + * an active poll. Cache size is not used as a fallback because it does not bound Java slot + * acquisition. + * *

Counts only outstanding polls, not executing tasks or pending reservations. Like Core's * balancer, this policy does not guarantee that both queues always have an outstanding poll. */ @ThreadSafe final class WorkflowPollerBalancer { - private final int maxSlots; + private final Optional maxSlots; private int stickyPolls; private int normalPolls; private int stickyTarget; @@ -19,7 +23,7 @@ final class WorkflowPollerBalancer { private boolean normalPollingDisabled; WorkflowPollerBalancer(Optional maxSlots, int initialStickyTarget) { - this.maxSlots = maxSlots.orElse(Integer.MAX_VALUE); + this.maxSlots = maxSlots; this.stickyTarget = initialStickyTarget; } @@ -39,7 +43,7 @@ private boolean canPoll(boolean sticky) { if (stickyPolls >= stickyTarget) { return false; } - if (!normalPollingDisabled && normalPolls == 0 && stickyPolls + 1 >= maxSlots) { + if (!normalPollingDisabled && normalPolls == 0 && wouldPreventFirstPoll(stickyPolls)) { return false; } if (preferSticky) { @@ -50,14 +54,18 @@ private boolean canPoll(boolean sticky) { if (normalPollingDisabled || normalPolls == 0) { return true; } - if (stickyPolls == 0 && normalPolls + 1 >= maxSlots) { + if (stickyPolls == 0 && wouldPreventFirstPoll(normalPolls)) { return false; } if (preferSticky) { return false; } } - return (long) stickyPolls + normalPolls < maxSlots; + return !maxSlots.isPresent() || (long) stickyPolls + normalPolls < maxSlots.get(); + } + + private boolean wouldPreventFirstPoll(int activePolls) { + return !maxSlots.isPresent() || activePolls + 1 >= maxSlots.get(); } synchronized void pollStarted(boolean sticky) { diff --git a/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java b/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java index f8e6df990f..a10aff8ddd 100644 --- a/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java +++ b/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java @@ -12,7 +12,8 @@ * *

Sticky and non-sticky workflow queues scale independently while sharing execution slots. * Admission prioritizes a meaningful sticky backlog while the sticky pool is below its target, - * without blocking the first poll of either queue type. + * without blocking the first poll of either queue type. If the slot supplier does not advertise a + * maximum, additional polls of one queue type wait until the other has an active poll. */ public final class PollerBehaviorAutoscaling implements PollerBehavior { private final int minConcurrentTaskPollers; diff --git a/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java b/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java index c1922a7985..2917438f45 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java @@ -7,9 +7,12 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.ByteString; import com.uber.m3.tally.NoopScope; import io.grpc.Status; +import io.temporal.api.enums.v1.TaskQueueKind; import io.temporal.api.workflowservice.v1.GetSystemInfoResponse; +import io.temporal.api.workflowservice.v1.PollWorkflowTaskQueueRequest; import io.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse; import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -17,6 +20,11 @@ import java.time.Duration; import java.util.Arrays; import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; @@ -102,6 +110,202 @@ public void stickyPoolDoesNotReserveAnExtraSlotAtItsTarget() throws Exception { assertEventually(Duration.ofSeconds(5), () -> assertEquals(0, slots.getIssuedSlots())); } + @Test(timeout = 30000) + public void stickyBacklogGrowsIntoSlotsReleasedByNormalPolls() throws Exception { + WorkflowServiceStubs service = mockService(); + BlockingQueue> normalResponses = + new LinkedBlockingQueue<>(); + BlockingQueue> stickyResponses = + new LinkedBlockingQueue<>(); + Queue> allResponses = + new ConcurrentLinkedQueue<>(); + when(service.futureStub().pollWorkflowTaskQueue(any())) + .thenAnswer( + invocation -> { + PollWorkflowTaskQueueRequest request = invocation.getArgument(0); + SettableFuture response = SettableFuture.create(); + allResponses.add(response); + (request.getTaskQueue().getKind() == TaskQueueKind.TASK_QUEUE_KIND_STICKY + ? stickyResponses + : normalResponses) + .add(response); + return response; + }); + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(new FixedSizeSlotSupplier<>(4), new NoopScope()); + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.of(4), 3); + AsyncWorkflowPollTask sticky = spy(pollTask(service, slots, true, balancer)); + AsyncWorkflowPollTask normal = spy(pollTask(service, slots, false, balancer)); + CountDownLatch startSticky = new CountDownLatch(1); + AtomicReference normalThread = new AtomicReference<>(); + capturePollThread(normal, normalThread); + doAnswer( + invocation -> { + startSticky.await(); + return invocation.callRealMethod(); + }) + .when(sticky) + .awaitPollAdmission(); + ShutdownableTaskExecutor executor = mock(ShutdownableTaskExecutor.class); + doAnswer( + invocation -> { + WorkflowTask task = invocation.getArgument(0); + task.getCompletionCallback().apply(SlotReleaseReason.taskComplete()); + return null; + }) + .when(executor) + .process(any()); + AsyncPoller poller = + new AsyncPoller<>( + slots, + new SlotReservationData("queue", "identity", ""), + Arrays.asList(normal, sticky), + executor, + PollerOptions.newBuilder() + .setPollThreadNamePrefix("sticky-recovery-test") + .setPollerBehavior(new PollerBehaviorAutoscaling(1, 3, 3)) + .build(), + new NamespaceCapabilities(), + new NoopScope()); + poller.start(); + try { + // Hold back sticky startup so normal polls occupy three of the four shared slots. + SettableFuture firstNormal = awaitPoll(normalResponses); + SettableFuture secondNormal = awaitPoll(normalResponses); + awaitPoll(normalResponses); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(normal, times(3)).onPollStarted(); + assertEquals(Thread.State.WAITING, normalThread.get().getState()); + assertEquals(3, slots.getIssuedSlots()); + }); + startSticky.countDown(); + SettableFuture firstSticky = awaitPoll(stickyResponses); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(sticky, times(1)).onPollStarted(); + assertEquals(4, slots.getIssuedSlots()); + }); + + // Introduce backlog through a real sticky task response, not by changing balancer state. + firstSticky.set( + PollWorkflowTaskQueueResponse.newBuilder() + .setTaskToken(ByteString.copyFromUtf8("sticky-task")) + .setBacklogCountHint(10) + .build()); + awaitPoll(stickyResponses); + verify(executor, timeout(5000)).process(any()); + + // Keep one normal poll outstanding. Each released slot must grow the sticky pool, + // rather than being reclaimed by another normal poll. + firstNormal.set(PollWorkflowTaskQueueResponse.getDefaultInstance()); + awaitPoll(stickyResponses); + secondNormal.set(PollWorkflowTaskQueueResponse.getDefaultInstance()); + awaitPoll(stickyResponses); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(sticky, times(4)).onPollStarted(); + verify(sticky, times(1)).onPollFinished(); + verify(normal, times(3)).onPollStarted(); + verify(normal, times(2)).onPollFinished(); + assertTrue("Normal polls must not reclaim released slots", normalResponses.isEmpty()); + assertEquals(4, slots.getIssuedSlots()); + }); + } finally { + startSticky.countDown(); + poller.shutdown(new ShutdownManager(), true).get(5, TimeUnit.SECONDS); + for (SettableFuture response : allResponses) { + response.setException(Status.CANCELLED.asRuntimeException()); + } + poller.awaitTermination(5, TimeUnit.SECONDS); + } + assertEventually(Duration.ofSeconds(5), () -> assertEquals(0, slots.getIssuedSlots())); + } + + @Test(timeout = 20000) + public void unknownCapacityLeavesRoomForFirstPollOfEitherQueue() throws Exception { + assertFirstPollAccessWithoutAdvertisedMaximum(true); + assertFirstPollAccessWithoutAdvertisedMaximum(false); + } + + private static void assertFirstPollAccessWithoutAdvertisedMaximum(boolean stickyFirst) + throws Exception { + WorkflowServiceStubs service = mockService(); + SettableFuture response = SettableFuture.create(); + when(service.futureStub().pollWorkflowTaskQueue(any())).thenReturn(response); + // The supplier has two real slots but does not advertise that limit to the balancer. + FixedSizeSlotSupplier supplier = + new FixedSizeSlotSupplier(2) { + @Override + public Optional getMaximumSlots() { + return Optional.empty(); + } + }; + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(supplier, new NoopScope()); + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(slots.maximumSlots(), 3); + AsyncWorkflowPollTask first = spy(pollTask(service, slots, stickyFirst, balancer)); + AsyncWorkflowPollTask other = spy(pollTask(service, slots, !stickyFirst, balancer)); + AtomicReference firstThread = new AtomicReference<>(); + capturePollThread(first, firstThread); + CountDownLatch startOther = new CountDownLatch(1); + doAnswer( + invocation -> { + startOther.await(); + return invocation.callRealMethod(); + }) + .when(other) + .awaitPollAdmission(); + AsyncPoller poller = + new AsyncPoller<>( + slots, + new SlotReservationData("queue", "identity", ""), + Arrays.asList(first, other), + mock(ShutdownableTaskExecutor.class), + PollerOptions.newBuilder() + .setPollThreadNamePrefix("unknown-capacity-test") + .setPollerBehavior(new PollerBehaviorAutoscaling(1, 3, 3)) + .build(), + new NamespaceCapabilities(), + new NoopScope()); + poller.start(); + try { + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(first, times(1)).onPollStarted(); + assertEquals(Thread.State.WAITING, firstThread.get().getState()); + assertEquals( + "Leave a slot for the other queue's first poll", 1, slots.getIssuedSlots()); + }); + startOther.countDown(); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(first, times(1)).onPollStarted(); + verify(other, times(1)).onPollStarted(); + assertEquals(2, slots.getIssuedSlots()); + }); + } finally { + startOther.countDown(); + poller.shutdown(new ShutdownManager(), true).get(5, TimeUnit.SECONDS); + response.setException(Status.CANCELLED.asRuntimeException()); + poller.awaitTermination(5, TimeUnit.SECONDS); + } + assertEventually(Duration.ofSeconds(5), () -> assertEquals(0, slots.getIssuedSlots())); + } + + private static SettableFuture awaitPoll( + BlockingQueue> responses) + throws InterruptedException { + SettableFuture response = responses.poll(5, TimeUnit.SECONDS); + assertNotNull("Expected another poll", response); + return response; + } + private static void capturePollThread(AsyncWorkflowPollTask task, AtomicReference thread) throws Exception { doAnswer( diff --git a/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java b/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java index b8e989fa67..ad1560ae80 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java @@ -108,6 +108,50 @@ public void unknownCapacityStillRespectsBacklogAndTarget() throws Exception { assertBlocked(balancer, true); } + @Test(timeout = 10000) + public void unknownCapacityWaitsForOtherQueuesFirstPoll() throws Exception { + for (boolean sticky : new boolean[] {true, false}) { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 3); + balancer.pollStarted(sticky); + if (sticky) { + balancer.recordStickyBacklog(10); + } + balancer.awaitAdmission(!sticky); + assertUnblockedBy(balancer, sticky, () -> balancer.pollStarted(!sticky)); + } + } + + @Test(timeout = 10000) + public void unknownCapacityRechecksWhenOtherQueueLosesItsLastPoll() throws Exception { + for (boolean sticky : new boolean[] {true, false}) { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 3); + balancer.pollStarted(sticky); + balancer.pollStarted(!sticky); + balancer.awaitAdmission(sticky); + balancer.pollFinished(!sticky); + assertBlocked(balancer, sticky); + balancer.awaitAdmission(!sticky); + } + } + + @Test(timeout = 10000) + public void unknownCapacityDoesNotImposeAFixedPoolSizeOnceBothQueuesHavePolls() throws Exception { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 5); + for (int i = 0; i < 3; i++) { + balancer.pollStarted(true); + balancer.pollStarted(false); + } + balancer.awaitAdmission(true); + balancer.awaitAdmission(false); + } + + @Test(timeout = 10000) + public void unknownCapacityStillAllowsStickyDrain() throws Exception { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 3); + balancer.pollStarted(true); + assertUnblockedBy(balancer, true, balancer::disableNormalPoll); + } + @Test(timeout = 10000) public void disablingNormalPollingReleasesCapacityForStickyDrain() throws Exception { WorkflowPollerBalancer balancer = balancer(2, 3, 1, 0);