diff --git a/CHANGELOG.md b/CHANGELOG.md index 9153295fe1..008ad40228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,10 @@ to docs, or any other relevant information. project-scoped custom roles, and caller rate limits. ### Fixed +- Autoscaling workflow pollers now prioritize meaningful sticky backlog up to the sticky pool's + target and use advertised slot capacity to balance sticky and non-sticky admission. Without an + advertised maximum, additional polls of one queue type wait until the other has an active poll. + Unlike Core, Java does not use workflow cache size as a fallback slot bound. - `toString()` of `WorkflowServiceStubsOptions`, `OperatorServiceStubsOptions` and `CloudServiceStubsOptions` now includes the fields inherited from `ServiceStubsOptions`. gRPC headers are rendered by name only, so values such as credentials set through `setHeaders` no longer appear in logs. diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java index 13f25e6c7a..cf79f70208 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncPoller.java @@ -11,11 +11,7 @@ import io.temporal.worker.tuning.SlotSupplierFuture; import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.IntSupplier; -import javax.annotation.concurrent.ThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,15 +23,13 @@ final class AsyncPoller extends BasePoller { private static final Logger log = LoggerFactory.getLogger(AsyncPoller.class); private final TrackingSlotSupplier slotSupplier; private final SlotReservationData slotReservationData; - private final List> asyncTaskPollers; + private final List> asyncTaskPollers; private final PollerOptions pollerOptions; private final PollerBehaviorAutoscaling pollerBehavior; private final Scope workerMetricsScope; private Throttler pollRateThrottler; private final Thread.UncaughtExceptionHandler uncaughtExceptionHandler = new PollerUncaughtExceptionHandler(); - private final PollQueueBalancer pollerBalancer = - new PollQueueBalancer(); // Used to balance the number of slots across pollers AsyncPoller( TrackingSlotSupplier slotSupplier, @@ -58,7 +52,7 @@ final class AsyncPoller extends BasePoller { AsyncPoller( TrackingSlotSupplier slotSupplier, SlotReservationData slotReservationData, - List> asyncTaskPollers, + List> asyncTaskPollers, ShutdownableTaskExecutor taskExecutor, PollerOptions pollerOptions, NamespaceCapabilities namespaceCapabilities, @@ -120,11 +114,12 @@ public boolean start() { asyncTaskPoller.getLabel(), newTarget); pollerSemaphore.setMaxPermits(newTarget); + asyncTaskPoller.onPollerTargetChanged(newTarget); }); + asyncTaskPoller.onPollerTargetChanged(pollerBehavior.getInitialConcurrentTaskPollers()); asyncTaskPoller.setPollerTargetSupplier(pollScaleReportHandle::getTargetPollerCount); PollQueueTask pollQueue = new PollQueueTask(asyncTaskPoller, pollerSemaphore, pollScaleReportHandle); - pollerBalancer.addPoller(asyncTaskPoller.getLabel()); exec.execute(pollQueue); exec.scheduleAtFixedRate(pollScaleReportHandle, 0, 100, TimeUnit.MILLISECONDS); } @@ -172,6 +167,23 @@ default String getLabel() { default void setPollerTargetSupplier(IntSupplier pollerTargetSupplier) { // no-op } + + /** Waits for admission before reserving an execution slot. */ + default void awaitPollAdmission() throws InterruptedException { + // no-op + } + + default void onPollStarted() { + // no-op + } + + default void onPollFinished() { + // no-op + } + + default void onPollerTargetChanged(int target) { + // no-op + } } class PollQueueTask implements Runnable { @@ -229,7 +241,7 @@ public void run() { continue; } - pollerBalancer.balance(asyncTaskPoller.getLabel()); + asyncTaskPoller.awaitPollAdmission(); if (shouldTerminate()) { continue; } @@ -257,12 +269,12 @@ public void run() { CompletableFuture pollRequest = asyncTaskPoller.poll(permit); // Mark that we have made a poll request pollRequestMade = true; - pollerBalancer.startPoll(asyncTaskPoller.getLabel()); + asyncTaskPoller.onPollStarted(); pollRequest .handle( (task, e) -> { - pollerBalancer.endPoll(asyncTaskPoller.getLabel()); + asyncTaskPoller.onPollFinished(); if (e instanceof CompletionException) { e = e.getCause(); } @@ -310,7 +322,6 @@ public void run() { } if (shouldTerminate()) { - pollerBalancer.removePoller(asyncTaskPoller.getLabel()); abort = true; log.debug( "Poll loop is terminated: {} - {}", @@ -321,80 +332,4 @@ public void run() { } } } - - /** - * PollQueueBalancer is used to ensure that at least one poll request is running for each task - * type. This is necessary to avoid one task type from consuming all the slots and starving other - * pollers. - */ - @ThreadSafe - class PollQueueBalancer { - Map taskCounts = new HashMap<>(); - private final Lock balancerLock = new ReentrantLock(); - private final Condition balancerCondition = balancerLock.newCondition(); - - void startPoll(String pollerName) { - balancerLock.lock(); - Integer currentPolls = taskCounts.compute(pollerName, (k, v) -> v + 1); - if (currentPolls == 1) { - balancerCondition.signalAll(); - } - balancerLock.unlock(); - } - - void endPoll(String pollerName) { - balancerLock.lock(); - if (!taskCounts.containsKey(pollerName)) { - balancerLock.unlock(); - return; - } - Integer currentPolls = taskCounts.compute(pollerName, (k, v) -> v - 1); - if (currentPolls == 0) { - balancerCondition.signalAll(); - } - balancerLock.unlock(); - } - - void addPoller(String pollerName) { - balancerLock.lock(); - taskCounts.put(pollerName, 0); - balancerCondition.signalAll(); - balancerLock.unlock(); - } - - void removePoller(String pollerName) { - balancerLock.lock(); - taskCounts.remove(pollerName); - balancerCondition.signalAll(); - balancerLock.unlock(); - } - - /** Ensure that at least one poller is running for each task. */ - void balance(String p) throws InterruptedException { - while (!shouldTerminate()) { - balancerLock.lock(); - try { - // If this poller has no tasks then we can unblock immediately - if (taskCounts.get(p) == 0) { - return; - } - // Check if all tasks have at least one poll request - boolean allOtherTasksHavePolls = true; - for (String task : taskCounts.keySet()) { - if (!Objects.equals(task, p) && taskCounts.get(task) == 0) { - allOtherTasksHavePolls = false; - break; - } - } - if (!allOtherTasksHavePolls) { - balancerCondition.await(); - } else { - return; - } - } finally { - balancerLock.unlock(); - } - } - } - } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java index 085cb931f3..7a152eca3b 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/AsyncWorkflowPollTask.java @@ -41,6 +41,7 @@ public class AsyncWorkflowPollTask private final Context.CancellableContext grpcContext = Context.ROOT.withCancellation(); private final AtomicBoolean shutdown = new AtomicBoolean(false); private final PollerTracker pollerTracker; + @Nullable private WorkflowPollerBalancer pollerBalancer; @Override public String toString() { @@ -120,6 +121,38 @@ public AsyncWorkflowPollTask( } } + void setPollerBalancer(WorkflowPollerBalancer pollerBalancer) { + this.pollerBalancer = pollerBalancer; + } + + @Override + public void awaitPollAdmission() throws InterruptedException { + if (pollerBalancer != null) { + pollerBalancer.awaitAdmission(stickyPoller); + } + } + + @Override + public void onPollStarted() { + if (pollerBalancer != null) { + pollerBalancer.pollStarted(stickyPoller); + } + } + + @Override + public void onPollFinished() { + if (pollerBalancer != null) { + pollerBalancer.pollFinished(stickyPoller); + } + } + + @Override + public void onPollerTargetChanged(int target) { + if (stickyPoller && pollerBalancer != null) { + pollerBalancer.setStickyTarget(target); + } + } + /** * Supplies the poller pool's current scaling target. Installed by AsyncPoller, which owns the * scale report handle; null until then, and for a pool that is not autoscaling. @@ -180,6 +213,9 @@ public CompletableFuture poll(SlotPermit permit) return response .thenApply( r -> { + if (stickyPoller && pollerBalancer != null && r != null) { + pollerBalancer.recordStickyBacklog(r.getBacklogCountHint()); + } if (r == null || r.getTaskToken().isEmpty()) { pollerMetricScope .counter(MetricsType.WORKFLOW_TASK_QUEUE_POLL_EMPTY_COUNTER) @@ -215,6 +251,9 @@ public void disableNormalPoll() { throw new IllegalStateException("Cannot disable normal poll for sticky poller"); } shutdown.set(true); + if (pollerBalancer != null) { + pollerBalancer.disableNormalPoll(); + } } @Override diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java new file mode 100644 index 0000000000..a56b14e5a8 --- /dev/null +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowPollerBalancer.java @@ -0,0 +1,103 @@ +package io.temporal.internal.worker; + +import java.util.Optional; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Coordinates autoscaling sticky and non-sticky poll admission before slot reservation. + * + *

Without an advertised slot maximum, additional polls of one type wait until the other type has + * an active poll. Cache size is not used as a fallback because it does not bound Java slot + * acquisition. + * + *

Counts only outstanding polls, not executing tasks or pending reservations. Like Core's + * balancer, this policy does not guarantee that both queues always have an outstanding poll. + */ +@ThreadSafe +final class WorkflowPollerBalancer { + private final Optional maxSlots; + private int stickyPolls; + private int normalPolls; + private int stickyTarget; + private long stickyBacklog; + private boolean normalPollingDisabled; + + WorkflowPollerBalancer(Optional maxSlots, int initialStickyTarget) { + this.maxSlots = maxSlots; + this.stickyTarget = initialStickyTarget; + } + + synchronized void awaitAdmission(boolean sticky) throws InterruptedException { + while (!canPoll(sticky)) { + wait(); + } + } + + private boolean canPoll(boolean sticky) { + boolean preferSticky = + stickyBacklog > 1 && stickyBacklog > stickyPolls && stickyPolls < stickyTarget; + if (sticky) { + if (stickyPolls == 0) { + return true; + } + if (stickyPolls >= stickyTarget) { + return false; + } + if (!normalPollingDisabled && normalPolls == 0 && wouldPreventFirstPoll(stickyPolls)) { + return false; + } + if (preferSticky) { + return true; + } + } else { + // Let a disabled normal poller reach poll(), which aborts without issuing an RPC. + if (normalPollingDisabled || normalPolls == 0) { + return true; + } + if (stickyPolls == 0 && wouldPreventFirstPoll(normalPolls)) { + return false; + } + if (preferSticky) { + return false; + } + } + return !maxSlots.isPresent() || (long) stickyPolls + normalPolls < maxSlots.get(); + } + + private boolean wouldPreventFirstPoll(int activePolls) { + return !maxSlots.isPresent() || activePolls + 1 >= maxSlots.get(); + } + + synchronized void pollStarted(boolean sticky) { + if (sticky) { + stickyPolls++; + } else { + normalPolls++; + } + notifyAll(); + } + + synchronized void pollFinished(boolean sticky) { + if (sticky) { + stickyPolls--; + } else { + normalPolls--; + } + notifyAll(); + } + + synchronized void recordStickyBacklog(long backlog) { + stickyBacklog = backlog; + notifyAll(); + } + + synchronized void setStickyTarget(int target) { + stickyTarget = target; + notifyAll(); + } + + synchronized void disableNormalPoll() { + normalPollingDisabled = true; + notifyAll(); + } +} diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java index 26c52ef27f..86105fb522 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java @@ -145,7 +145,7 @@ public boolean start() { boolean useAsyncPoller = pollerOptions.getPollerBehavior() instanceof PollerBehaviorAutoscaling; if (useAsyncPoller) { - List> pollers; + List pollers; if (stickyTaskQueueName != null) { AsyncWorkflowPollTask normalPoller = new AsyncWorkflowPollTask( @@ -195,6 +195,16 @@ public boolean start() { pollerTracker, workerControlTaskQueue())); } + if (stickyTaskQueueName != null) { + WorkflowPollerBalancer balancer = + new WorkflowPollerBalancer( + slotSupplier.maximumSlots(), + ((PollerBehaviorAutoscaling) pollerOptions.getPollerBehavior()) + .getInitialConcurrentTaskPollers()); + for (AsyncWorkflowPollTask task : pollers) { + task.setPollerBalancer(balancer); + } + } poller = new AsyncPoller<>( slotSupplier, diff --git a/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java b/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java index 7687c39350..a10aff8ddd 100644 --- a/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java +++ b/temporal-sdk/src/main/java/io/temporal/worker/tuning/PollerBehaviorAutoscaling.java @@ -9,6 +9,11 @@ * *

If the server does not support autoscaling, then the number of pollers will stay at the * initial number of pollers. + * + *

Sticky and non-sticky workflow queues scale independently while sharing execution slots. + * Admission prioritizes a meaningful sticky backlog while the sticky pool is below its target, + * without blocking the first poll of either queue type. If the slot supplier does not advertise a + * maximum, additional polls of one queue type wait until the other has an active poll. */ public final class PollerBehaviorAutoscaling implements PollerBehavior { private final int minConcurrentTaskPollers; diff --git a/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java b/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java new file mode 100644 index 0000000000..2917438f45 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/worker/AsyncWorkflowPollerBalancingTest.java @@ -0,0 +1,351 @@ +package io.temporal.internal.worker; + +import static io.temporal.testUtils.Eventually.assertEventually; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.ByteString; +import com.uber.m3.tally.NoopScope; +import io.grpc.Status; +import io.temporal.api.enums.v1.TaskQueueKind; +import io.temporal.api.workflowservice.v1.GetSystemInfoResponse; +import io.temporal.api.workflowservice.v1.PollWorkflowTaskQueueRequest; +import io.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse; +import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.worker.tuning.*; +import java.time.Duration; +import java.util.Arrays; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +public class AsyncWorkflowPollerBalancingTest { + @Test(timeout = 10000) + public void onlyStickyResponsesUpdateBacklogIncludingEmptyResponses() throws Exception { + WorkflowServiceStubs service = mockService(); + when(service.futureStub().pollWorkflowTaskQueue(any())) + .thenReturn( + Futures.immediateFuture( + PollWorkflowTaskQueueResponse.newBuilder().setBacklogCountHint(10).build()), + Futures.immediateFuture( + PollWorkflowTaskQueueResponse.newBuilder().setBacklogCountHint(10).build()), + Futures.immediateFuture(PollWorkflowTaskQueueResponse.getDefaultInstance())); + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(new FixedSizeSlotSupplier<>(10), new NoopScope()); + WorkflowPollerBalancer balancer = spy(new WorkflowPollerBalancer(Optional.of(10), 3)); + AsyncWorkflowPollTask sticky = pollTask(service, slots, true, balancer); + AsyncWorkflowPollTask normal = pollTask(service, slots, false, balancer); + + normal.onPollerTargetChanged(1); + verify(balancer, never()).setStickyTarget(anyInt()); + sticky.onPollerTargetChanged(2); + verify(balancer).setStickyTarget(2); + + assertNull(normal.poll(new SlotPermit()).get(5, TimeUnit.SECONDS)); + verify(balancer, never()).recordStickyBacklog(anyLong()); + assertNull(sticky.poll(new SlotPermit()).get(5, TimeUnit.SECONDS)); + verify(balancer).recordStickyBacklog(10); + assertNull(sticky.poll(new SlotPermit()).get(5, TimeUnit.SECONDS)); + verify(balancer).recordStickyBacklog(0); + } + + @Test(timeout = 10000) + public void stickyPoolDoesNotReserveAnExtraSlotAtItsTarget() throws Exception { + WorkflowServiceStubs service = mockService(); + SettableFuture response = SettableFuture.create(); + when(service.futureStub().pollWorkflowTaskQueue(any())).thenReturn(response); + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(new FixedSizeSlotSupplier<>(10), new NoopScope()); + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.of(10), 1); + AtomicReference stickyThread = new AtomicReference<>(); + AtomicReference normalThread = new AtomicReference<>(); + AsyncWorkflowPollTask sticky = spy(pollTask(service, slots, true, balancer)); + AsyncWorkflowPollTask normal = spy(pollTask(service, slots, false, balancer)); + capturePollThread(sticky, stickyThread); + capturePollThread(normal, normalThread); + ShutdownableTaskExecutor executor = mock(ShutdownableTaskExecutor.class); + AsyncPoller poller = + new AsyncPoller<>( + slots, + new SlotReservationData("queue", "identity", ""), + Arrays.asList(sticky, normal), + executor, + PollerOptions.newBuilder() + .setPollThreadNamePrefix("balancing-test") + .setPollerBehavior(new PollerBehaviorAutoscaling(1, 3, 1)) + .build(), + new NamespaceCapabilities(), + new NoopScope()); + poller.start(); + try { + assertEventually( + Duration.ofSeconds(5), + () -> { + assertNotNull(stickyThread.get()); + assertNotNull(normalThread.get()); + assertEquals(Thread.State.WAITING, stickyThread.get().getState()); + assertEquals(Thread.State.WAITING, normalThread.get().getState()); + // One slot per outstanding poll, plus a normal slot waiting on poller concurrency. + // Sticky must wait for admission BEFORE reserving a second slot. + assertEquals(3, slots.getIssuedSlots()); + }); + verify(sticky, times(1)).onPollStarted(); + verify(normal, times(1)).onPollStarted(); + verify(service.futureStub(), times(2)).pollWorkflowTaskQueue(any()); + } finally { + poller.shutdown(new ShutdownManager(), true).get(5, TimeUnit.SECONDS); + response.setException(Status.CANCELLED.asRuntimeException()); + poller.awaitTermination(5, TimeUnit.SECONDS); + } + assertEventually(Duration.ofSeconds(5), () -> assertEquals(0, slots.getIssuedSlots())); + } + + @Test(timeout = 30000) + public void stickyBacklogGrowsIntoSlotsReleasedByNormalPolls() throws Exception { + WorkflowServiceStubs service = mockService(); + BlockingQueue> normalResponses = + new LinkedBlockingQueue<>(); + BlockingQueue> stickyResponses = + new LinkedBlockingQueue<>(); + Queue> allResponses = + new ConcurrentLinkedQueue<>(); + when(service.futureStub().pollWorkflowTaskQueue(any())) + .thenAnswer( + invocation -> { + PollWorkflowTaskQueueRequest request = invocation.getArgument(0); + SettableFuture response = SettableFuture.create(); + allResponses.add(response); + (request.getTaskQueue().getKind() == TaskQueueKind.TASK_QUEUE_KIND_STICKY + ? stickyResponses + : normalResponses) + .add(response); + return response; + }); + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(new FixedSizeSlotSupplier<>(4), new NoopScope()); + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.of(4), 3); + AsyncWorkflowPollTask sticky = spy(pollTask(service, slots, true, balancer)); + AsyncWorkflowPollTask normal = spy(pollTask(service, slots, false, balancer)); + CountDownLatch startSticky = new CountDownLatch(1); + AtomicReference normalThread = new AtomicReference<>(); + capturePollThread(normal, normalThread); + doAnswer( + invocation -> { + startSticky.await(); + return invocation.callRealMethod(); + }) + .when(sticky) + .awaitPollAdmission(); + ShutdownableTaskExecutor executor = mock(ShutdownableTaskExecutor.class); + doAnswer( + invocation -> { + WorkflowTask task = invocation.getArgument(0); + task.getCompletionCallback().apply(SlotReleaseReason.taskComplete()); + return null; + }) + .when(executor) + .process(any()); + AsyncPoller poller = + new AsyncPoller<>( + slots, + new SlotReservationData("queue", "identity", ""), + Arrays.asList(normal, sticky), + executor, + PollerOptions.newBuilder() + .setPollThreadNamePrefix("sticky-recovery-test") + .setPollerBehavior(new PollerBehaviorAutoscaling(1, 3, 3)) + .build(), + new NamespaceCapabilities(), + new NoopScope()); + poller.start(); + try { + // Hold back sticky startup so normal polls occupy three of the four shared slots. + SettableFuture firstNormal = awaitPoll(normalResponses); + SettableFuture secondNormal = awaitPoll(normalResponses); + awaitPoll(normalResponses); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(normal, times(3)).onPollStarted(); + assertEquals(Thread.State.WAITING, normalThread.get().getState()); + assertEquals(3, slots.getIssuedSlots()); + }); + startSticky.countDown(); + SettableFuture firstSticky = awaitPoll(stickyResponses); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(sticky, times(1)).onPollStarted(); + assertEquals(4, slots.getIssuedSlots()); + }); + + // Introduce backlog through a real sticky task response, not by changing balancer state. + firstSticky.set( + PollWorkflowTaskQueueResponse.newBuilder() + .setTaskToken(ByteString.copyFromUtf8("sticky-task")) + .setBacklogCountHint(10) + .build()); + awaitPoll(stickyResponses); + verify(executor, timeout(5000)).process(any()); + + // Keep one normal poll outstanding. Each released slot must grow the sticky pool, + // rather than being reclaimed by another normal poll. + firstNormal.set(PollWorkflowTaskQueueResponse.getDefaultInstance()); + awaitPoll(stickyResponses); + secondNormal.set(PollWorkflowTaskQueueResponse.getDefaultInstance()); + awaitPoll(stickyResponses); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(sticky, times(4)).onPollStarted(); + verify(sticky, times(1)).onPollFinished(); + verify(normal, times(3)).onPollStarted(); + verify(normal, times(2)).onPollFinished(); + assertTrue("Normal polls must not reclaim released slots", normalResponses.isEmpty()); + assertEquals(4, slots.getIssuedSlots()); + }); + } finally { + startSticky.countDown(); + poller.shutdown(new ShutdownManager(), true).get(5, TimeUnit.SECONDS); + for (SettableFuture response : allResponses) { + response.setException(Status.CANCELLED.asRuntimeException()); + } + poller.awaitTermination(5, TimeUnit.SECONDS); + } + assertEventually(Duration.ofSeconds(5), () -> assertEquals(0, slots.getIssuedSlots())); + } + + @Test(timeout = 20000) + public void unknownCapacityLeavesRoomForFirstPollOfEitherQueue() throws Exception { + assertFirstPollAccessWithoutAdvertisedMaximum(true); + assertFirstPollAccessWithoutAdvertisedMaximum(false); + } + + private static void assertFirstPollAccessWithoutAdvertisedMaximum(boolean stickyFirst) + throws Exception { + WorkflowServiceStubs service = mockService(); + SettableFuture response = SettableFuture.create(); + when(service.futureStub().pollWorkflowTaskQueue(any())).thenReturn(response); + // The supplier has two real slots but does not advertise that limit to the balancer. + FixedSizeSlotSupplier supplier = + new FixedSizeSlotSupplier(2) { + @Override + public Optional getMaximumSlots() { + return Optional.empty(); + } + }; + TrackingSlotSupplier slots = + new TrackingSlotSupplier<>(supplier, new NoopScope()); + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(slots.maximumSlots(), 3); + AsyncWorkflowPollTask first = spy(pollTask(service, slots, stickyFirst, balancer)); + AsyncWorkflowPollTask other = spy(pollTask(service, slots, !stickyFirst, balancer)); + AtomicReference firstThread = new AtomicReference<>(); + capturePollThread(first, firstThread); + CountDownLatch startOther = new CountDownLatch(1); + doAnswer( + invocation -> { + startOther.await(); + return invocation.callRealMethod(); + }) + .when(other) + .awaitPollAdmission(); + AsyncPoller poller = + new AsyncPoller<>( + slots, + new SlotReservationData("queue", "identity", ""), + Arrays.asList(first, other), + mock(ShutdownableTaskExecutor.class), + PollerOptions.newBuilder() + .setPollThreadNamePrefix("unknown-capacity-test") + .setPollerBehavior(new PollerBehaviorAutoscaling(1, 3, 3)) + .build(), + new NamespaceCapabilities(), + new NoopScope()); + poller.start(); + try { + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(first, times(1)).onPollStarted(); + assertEquals(Thread.State.WAITING, firstThread.get().getState()); + assertEquals( + "Leave a slot for the other queue's first poll", 1, slots.getIssuedSlots()); + }); + startOther.countDown(); + assertEventually( + Duration.ofSeconds(5), + () -> { + verify(first, times(1)).onPollStarted(); + verify(other, times(1)).onPollStarted(); + assertEquals(2, slots.getIssuedSlots()); + }); + } finally { + startOther.countDown(); + poller.shutdown(new ShutdownManager(), true).get(5, TimeUnit.SECONDS); + response.setException(Status.CANCELLED.asRuntimeException()); + poller.awaitTermination(5, TimeUnit.SECONDS); + } + assertEventually(Duration.ofSeconds(5), () -> assertEquals(0, slots.getIssuedSlots())); + } + + private static SettableFuture awaitPoll( + BlockingQueue> responses) + throws InterruptedException { + SettableFuture response = responses.poll(5, TimeUnit.SECONDS); + assertNotNull("Expected another poll", response); + return response; + } + + private static void capturePollThread(AsyncWorkflowPollTask task, AtomicReference thread) + throws Exception { + doAnswer( + invocation -> { + thread.set(Thread.currentThread()); + return invocation.callRealMethod(); + }) + .when(task) + .awaitPollAdmission(); + } + + private static WorkflowServiceStubs mockService() { + WorkflowServiceStubs service = mock(WorkflowServiceStubs.class); + WorkflowServiceGrpc.WorkflowServiceFutureStub stub = + mock(WorkflowServiceGrpc.WorkflowServiceFutureStub.class); + when(service.futureStub()).thenReturn(stub); + when(stub.withOption(any(), any())).thenReturn(stub); + return service; + } + + private static AsyncWorkflowPollTask pollTask( + WorkflowServiceStubs service, + TrackingSlotSupplier slots, + boolean sticky, + WorkflowPollerBalancer balancer) { + AsyncWorkflowPollTask task = + new AsyncWorkflowPollTask( + service, + "default", + "queue", + sticky ? "sticky-queue" : null, + "identity", + "instance-key", + new WorkerVersioningOptions("", false, null), + slots, + new NoopScope(), + GetSystemInfoResponse.Capabilities::getDefaultInstance, + new PollerTracker(), + null); + task.setPollerBalancer(balancer); + return task; + } +} diff --git a/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java b/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java new file mode 100644 index 0000000000..ad1560ae80 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/worker/WorkflowPollerBalancerTest.java @@ -0,0 +1,210 @@ +package io.temporal.internal.worker; + +import static io.temporal.testUtils.Eventually.assertEventually; +import static org.junit.Assert.*; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +public class WorkflowPollerBalancerTest { + @Test(timeout = 10000) + public void stickyPreferenceRequiresMeaningfulBacklog() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 3, 1, 1); + for (long backlog : new long[] {0, 1}) { + balancer.recordStickyBacklog(backlog); + balancer.awaitAdmission(false); + balancer.awaitAdmission(true); + } + balancer.recordStickyBacklog(10); + balancer.awaitAdmission(true); + assertBlocked(balancer, false); + + // A backlog no larger than the outstanding sticky polls is not meaningful. + balancer.pollStarted(true); + balancer.recordStickyBacklog(2); + balancer.awaitAdmission(false); + } + + @Test(timeout = 10000) + public void reachingStickyTargetReleasesPreference() throws Exception { + WorkflowPollerBalancer balancer = balancer(5, 3, 2, 1); + balancer.recordStickyBacklog(10); + assertUnblockedBy(balancer, false, () -> balancer.pollStarted(true)); + assertBlocked(balancer, true); + } + + @Test(timeout = 10000) + public void targetDecreaseWakesNormalWaiter() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 3, 2, 1); + balancer.recordStickyBacklog(10); + assertUnblockedBy(balancer, false, () -> balancer.setStickyTarget(2)); + assertBlocked(balancer, true); + } + + @Test(timeout = 10000) + public void targetIncreaseWakesStickyWaiter() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 1, 1, 1); + assertUnblockedBy(balancer, true, () -> balancer.setStickyTarget(2)); + } + + @Test(timeout = 10000) + public void backlogChangeWakesNormalWaiter() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 3, 1, 1); + balancer.recordStickyBacklog(10); + assertUnblockedBy(balancer, false, () -> balancer.recordStickyBacklog(1)); + } + + @Test(timeout = 10000) + public void pollCompletionWakesWaiterAtCapacity() throws Exception { + WorkflowPollerBalancer balancer = balancer(4, 4, 2, 2); + assertUnblockedBy(balancer, false, () -> balancer.pollFinished(true)); + balancer.pollStarted(true); + assertUnblockedBy(balancer, true, () -> balancer.pollFinished(false)); + } + + @Test(timeout = 10000) + public void firstPollOfEachTypeCanAttemptAdmission() throws Exception { + for (int capacity : new int[] {1, 2, 4}) { + WorkflowPollerBalancer balancer = balancer(capacity, 3, 0, 1); + balancer.awaitAdmission(true); + balancer = balancer(capacity, 3, 1, 0); + balancer.recordStickyBacklog(10); + balancer.awaitAdmission(false); + } + } + + @Test(timeout = 10000) + public void reserveRoomForOtherQueuesFirstPoll() throws Exception { + for (int capacity : new int[] {1, 2, 4}) { + assertBlocked(balancer(capacity, 10, Math.max(1, capacity - 1), 0), true); + assertBlocked(balancer(capacity, 10, 0, Math.max(1, capacity - 1)), false); + } + } + + @Test(timeout = 10000) + public void missingOtherPollDoesNotBlockSpareCapacity() throws Exception { + balancer(4, 3, 1, 0).awaitAdmission(true); + balancer(4, 3, 0, 1).awaitAdmission(false); + } + + @Test(timeout = 10000) + public void stickyTargetPreventsAdmissionEvenWithSpareSlots() throws Exception { + assertBlocked(balancer(10, 2, 2, 1), true); + assertBlocked(balancer(10, 1, 2, 1), true); + } + + @Test(timeout = 10000) + public void unknownCapacityStillRespectsBacklogAndTarget() throws Exception { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 2); + balancer.pollStarted(true); + balancer.pollStarted(false); + balancer.recordStickyBacklog(10); + assertBlocked(balancer, false); + balancer.pollStarted(true); + balancer.awaitAdmission(false); + assertBlocked(balancer, true); + } + + @Test(timeout = 10000) + public void unknownCapacityWaitsForOtherQueuesFirstPoll() throws Exception { + for (boolean sticky : new boolean[] {true, false}) { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 3); + balancer.pollStarted(sticky); + if (sticky) { + balancer.recordStickyBacklog(10); + } + balancer.awaitAdmission(!sticky); + assertUnblockedBy(balancer, sticky, () -> balancer.pollStarted(!sticky)); + } + } + + @Test(timeout = 10000) + public void unknownCapacityRechecksWhenOtherQueueLosesItsLastPoll() throws Exception { + for (boolean sticky : new boolean[] {true, false}) { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 3); + balancer.pollStarted(sticky); + balancer.pollStarted(!sticky); + balancer.awaitAdmission(sticky); + balancer.pollFinished(!sticky); + assertBlocked(balancer, sticky); + balancer.awaitAdmission(!sticky); + } + } + + @Test(timeout = 10000) + public void unknownCapacityDoesNotImposeAFixedPoolSizeOnceBothQueuesHavePolls() throws Exception { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 5); + for (int i = 0; i < 3; i++) { + balancer.pollStarted(true); + balancer.pollStarted(false); + } + balancer.awaitAdmission(true); + balancer.awaitAdmission(false); + } + + @Test(timeout = 10000) + public void unknownCapacityStillAllowsStickyDrain() throws Exception { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.empty(), 3); + balancer.pollStarted(true); + assertUnblockedBy(balancer, true, balancer::disableNormalPoll); + } + + @Test(timeout = 10000) + public void disablingNormalPollingReleasesCapacityForStickyDrain() throws Exception { + WorkflowPollerBalancer balancer = balancer(2, 3, 1, 0); + assertUnblockedBy(balancer, true, balancer::disableNormalPoll); + } + + private static WorkflowPollerBalancer balancer( + int slots, int target, int stickyPolls, int normalPolls) { + WorkflowPollerBalancer balancer = new WorkflowPollerBalancer(Optional.of(slots), target); + for (int i = 0; i < stickyPolls; i++) { + balancer.pollStarted(true); + } + for (int i = 0; i < normalPolls; i++) { + balancer.pollStarted(false); + } + return balancer; + } + + private static void assertBlocked(WorkflowPollerBalancer balancer, boolean sticky) + throws Exception { + assertWaiter(balancer, sticky, null); + } + + private static void assertUnblockedBy( + WorkflowPollerBalancer balancer, boolean sticky, Runnable change) throws Exception { + assertWaiter(balancer, sticky, change); + } + + private static void assertWaiter(WorkflowPollerBalancer balancer, boolean sticky, Runnable change) + throws Exception { + FutureTask admission = + new FutureTask<>( + () -> { + balancer.awaitAdmission(sticky); + return null; + }); + Thread thread = new Thread(admission, "balancer-test"); + thread.start(); + try { + assertEventually( + Duration.ofSeconds(5), + () -> { + assertFalse("Admission should be blocked", admission.isDone()); + assertEquals(Thread.State.WAITING, thread.getState()); + }); + if (change != null) { + change.run(); + admission.get(5, TimeUnit.SECONDS); + } + } finally { + thread.interrupt(); + thread.join(5000); + assertFalse("Admission waiter should terminate on interrupt", thread.isAlive()); + } + } +}