Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ to docs, or any other relevant information.
project-scoped custom roles, and caller rate limits.

### Fixed
- Autoscaling workflow pollers now prioritize meaningful sticky backlog up to the sticky pool's
target and use advertised slot capacity to balance sticky and non-sticky admission. Without an
advertised maximum, additional polls of one queue type wait until the other has an active poll.
Unlike Core, Java does not use workflow cache size as a fallback slot bound.
- `toString()` of `WorkflowServiceStubsOptions`, `OperatorServiceStubsOptions` and `CloudServiceStubsOptions` now
includes the fields inherited from `ServiceStubsOptions`. gRPC headers are rendered by name only, so values such as
credentials set through `setHeaders` no longer appear in logs.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,7 @@
import io.temporal.worker.tuning.SlotSupplierFuture;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.IntSupplier;
import javax.annotation.concurrent.ThreadSafe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -27,15 +23,13 @@ final class AsyncPoller<T extends ScalingTask> extends BasePoller<T> {
private static final Logger log = LoggerFactory.getLogger(AsyncPoller.class);
private final TrackingSlotSupplier<?> slotSupplier;
private final SlotReservationData slotReservationData;
private final List<PollTaskAsync<T>> asyncTaskPollers;
private final List<? extends PollTaskAsync<T>> asyncTaskPollers;
private final PollerOptions pollerOptions;
private final PollerBehaviorAutoscaling pollerBehavior;
private final Scope workerMetricsScope;
private Throttler pollRateThrottler;
private final Thread.UncaughtExceptionHandler uncaughtExceptionHandler =
new PollerUncaughtExceptionHandler();
private final PollQueueBalancer pollerBalancer =
new PollQueueBalancer(); // Used to balance the number of slots across pollers

AsyncPoller(
TrackingSlotSupplier<?> slotSupplier,
Expand All @@ -58,7 +52,7 @@ final class AsyncPoller<T extends ScalingTask> extends BasePoller<T> {
AsyncPoller(
TrackingSlotSupplier<?> slotSupplier,
SlotReservationData slotReservationData,
List<PollTaskAsync<T>> asyncTaskPollers,
List<? extends PollTaskAsync<T>> asyncTaskPollers,
ShutdownableTaskExecutor<T> taskExecutor,
PollerOptions pollerOptions,
NamespaceCapabilities namespaceCapabilities,
Expand Down Expand Up @@ -120,11 +114,12 @@ public boolean start() {
asyncTaskPoller.getLabel(),
newTarget);
pollerSemaphore.setMaxPermits(newTarget);
asyncTaskPoller.onPollerTargetChanged(newTarget);
});
asyncTaskPoller.onPollerTargetChanged(pollerBehavior.getInitialConcurrentTaskPollers());
asyncTaskPoller.setPollerTargetSupplier(pollScaleReportHandle::getTargetPollerCount);
PollQueueTask pollQueue =
new PollQueueTask(asyncTaskPoller, pollerSemaphore, pollScaleReportHandle);
pollerBalancer.addPoller(asyncTaskPoller.getLabel());
exec.execute(pollQueue);
exec.scheduleAtFixedRate(pollScaleReportHandle, 0, 100, TimeUnit.MILLISECONDS);
}
Expand Down Expand Up @@ -172,6 +167,23 @@ default String getLabel() {
default void setPollerTargetSupplier(IntSupplier pollerTargetSupplier) {
// no-op
}

/** Waits for admission before reserving an execution slot. */
default void awaitPollAdmission() throws InterruptedException {
// no-op
}

default void onPollStarted() {
// no-op
}

default void onPollFinished() {
// no-op
}

default void onPollerTargetChanged(int target) {
// no-op
}
}

class PollQueueTask implements Runnable {
Expand Down Expand Up @@ -229,7 +241,7 @@ public void run() {
continue;
}

pollerBalancer.balance(asyncTaskPoller.getLabel());
asyncTaskPoller.awaitPollAdmission();
if (shouldTerminate()) {
continue;
}
Expand Down Expand Up @@ -257,12 +269,12 @@ public void run() {
CompletableFuture<T> pollRequest = asyncTaskPoller.poll(permit);
// Mark that we have made a poll request
pollRequestMade = true;
pollerBalancer.startPoll(asyncTaskPoller.getLabel());
asyncTaskPoller.onPollStarted();

pollRequest
.handle(
(task, e) -> {
pollerBalancer.endPoll(asyncTaskPoller.getLabel());
asyncTaskPoller.onPollFinished();
if (e instanceof CompletionException) {
e = e.getCause();
}
Expand Down Expand Up @@ -310,7 +322,6 @@ public void run() {
}

if (shouldTerminate()) {
pollerBalancer.removePoller(asyncTaskPoller.getLabel());
abort = true;
log.debug(
"Poll loop is terminated: {} - {}",
Expand All @@ -321,80 +332,4 @@ public void run() {
}
}
}

/**
* PollQueueBalancer is used to ensure that at least one poll request is running for each task
* type. This is necessary to avoid one task type from consuming all the slots and starving other
* pollers.
*/
@ThreadSafe
class PollQueueBalancer {
Map<String, Integer> taskCounts = new HashMap<>();
private final Lock balancerLock = new ReentrantLock();
private final Condition balancerCondition = balancerLock.newCondition();

void startPoll(String pollerName) {
balancerLock.lock();
Integer currentPolls = taskCounts.compute(pollerName, (k, v) -> v + 1);
if (currentPolls == 1) {
balancerCondition.signalAll();
}
balancerLock.unlock();
}

void endPoll(String pollerName) {
balancerLock.lock();
if (!taskCounts.containsKey(pollerName)) {
balancerLock.unlock();
return;
}
Integer currentPolls = taskCounts.compute(pollerName, (k, v) -> v - 1);
if (currentPolls == 0) {
balancerCondition.signalAll();
}
balancerLock.unlock();
}

void addPoller(String pollerName) {
balancerLock.lock();
taskCounts.put(pollerName, 0);
balancerCondition.signalAll();
balancerLock.unlock();
}

void removePoller(String pollerName) {
balancerLock.lock();
taskCounts.remove(pollerName);
balancerCondition.signalAll();
balancerLock.unlock();
}

/** Ensure that at least one poller is running for each task. */
void balance(String p) throws InterruptedException {
while (!shouldTerminate()) {
balancerLock.lock();
try {
// If this poller has no tasks then we can unblock immediately
if (taskCounts.get(p) == 0) {
return;
}
// Check if all tasks have at least one poll request
boolean allOtherTasksHavePolls = true;
for (String task : taskCounts.keySet()) {
if (!Objects.equals(task, p) && taskCounts.get(task) == 0) {
allOtherTasksHavePolls = false;
break;
}
}
if (!allOtherTasksHavePolls) {
balancerCondition.await();
} else {
return;
}
} finally {
balancerLock.unlock();
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public class AsyncWorkflowPollTask
private final Context.CancellableContext grpcContext = Context.ROOT.withCancellation();
private final AtomicBoolean shutdown = new AtomicBoolean(false);
private final PollerTracker pollerTracker;
@Nullable private WorkflowPollerBalancer pollerBalancer;

@Override
public String toString() {
Expand Down Expand Up @@ -120,6 +121,38 @@ public AsyncWorkflowPollTask(
}
}

void setPollerBalancer(WorkflowPollerBalancer pollerBalancer) {
this.pollerBalancer = pollerBalancer;
}

@Override
public void awaitPollAdmission() throws InterruptedException {
if (pollerBalancer != null) {
pollerBalancer.awaitAdmission(stickyPoller);
}
}

@Override
public void onPollStarted() {
if (pollerBalancer != null) {
pollerBalancer.pollStarted(stickyPoller);
}
}

@Override
public void onPollFinished() {
if (pollerBalancer != null) {
pollerBalancer.pollFinished(stickyPoller);
}
}

@Override
public void onPollerTargetChanged(int target) {
if (stickyPoller && pollerBalancer != null) {
pollerBalancer.setStickyTarget(target);
}
}

/**
* Supplies the poller pool's current scaling target. Installed by AsyncPoller, which owns the
* scale report handle; null until then, and for a pool that is not autoscaling.
Expand Down Expand Up @@ -180,6 +213,9 @@ public CompletableFuture<WorkflowTask> poll(SlotPermit permit)
return response
.thenApply(
r -> {
if (stickyPoller && pollerBalancer != null && r != null) {
pollerBalancer.recordStickyBacklog(r.getBacklogCountHint());
}
if (r == null || r.getTaskToken().isEmpty()) {
pollerMetricScope
.counter(MetricsType.WORKFLOW_TASK_QUEUE_POLL_EMPTY_COUNTER)
Expand Down Expand Up @@ -215,6 +251,9 @@ public void disableNormalPoll() {
throw new IllegalStateException("Cannot disable normal poll for sticky poller");
}
shutdown.set(true);
if (pollerBalancer != null) {
pollerBalancer.disableNormalPoll();
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package io.temporal.internal.worker;

import java.util.Optional;
import javax.annotation.concurrent.ThreadSafe;

/**
* Coordinates autoscaling sticky and non-sticky poll admission before slot reservation.
*
* <p>Without an advertised slot maximum, additional polls of one type wait until the other type has
* an active poll. Cache size is not used as a fallback because it does not bound Java slot
* acquisition.
*
* <p>Counts only outstanding polls, not executing tasks or pending reservations. Like Core's
* balancer, this policy does not guarantee that both queues always have an outstanding poll.
*/
@ThreadSafe
final class WorkflowPollerBalancer {
private final Optional<Integer> maxSlots;
private int stickyPolls;
private int normalPolls;
private int stickyTarget;
private long stickyBacklog;
private boolean normalPollingDisabled;

WorkflowPollerBalancer(Optional<Integer> maxSlots, int initialStickyTarget) {
this.maxSlots = maxSlots;
this.stickyTarget = initialStickyTarget;
}

synchronized void awaitAdmission(boolean sticky) throws InterruptedException {
while (!canPoll(sticky)) {
wait();
}
}

private boolean canPoll(boolean sticky) {
boolean preferSticky =
stickyBacklog > 1 && stickyBacklog > stickyPolls && stickyPolls < stickyTarget;
if (sticky) {
if (stickyPolls == 0) {
return true;
}
if (stickyPolls >= stickyTarget) {
return false;
}
if (!normalPollingDisabled && normalPolls == 0 && wouldPreventFirstPoll(stickyPolls)) {
return false;
}
if (preferSticky) {
return true;
}
} else {
// Let a disabled normal poller reach poll(), which aborts without issuing an RPC.
if (normalPollingDisabled || normalPolls == 0) {
return true;
}
if (stickyPolls == 0 && wouldPreventFirstPoll(normalPolls)) {
return false;
}
if (preferSticky) {
return false;
}
}
return !maxSlots.isPresent() || (long) stickyPolls + normalPolls < maxSlots.get();
}

private boolean wouldPreventFirstPoll(int activePolls) {
return !maxSlots.isPresent() || activePolls + 1 >= maxSlots.get();
}

synchronized void pollStarted(boolean sticky) {
if (sticky) {
stickyPolls++;
} else {
normalPolls++;
}
notifyAll();
}

synchronized void pollFinished(boolean sticky) {
if (sticky) {
stickyPolls--;
} else {
normalPolls--;
}
notifyAll();
}

synchronized void recordStickyBacklog(long backlog) {
stickyBacklog = backlog;
notifyAll();
}

synchronized void setStickyTarget(int target) {
stickyTarget = target;
notifyAll();
}

synchronized void disableNormalPoll() {
normalPollingDisabled = true;
notifyAll();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ public boolean start() {
boolean useAsyncPoller =
pollerOptions.getPollerBehavior() instanceof PollerBehaviorAutoscaling;
if (useAsyncPoller) {
List<AsyncPoller.PollTaskAsync<WorkflowTask>> pollers;
List<AsyncWorkflowPollTask> pollers;
if (stickyTaskQueueName != null) {
AsyncWorkflowPollTask normalPoller =
new AsyncWorkflowPollTask(
Expand Down Expand Up @@ -195,6 +195,16 @@ public boolean start() {
pollerTracker,
workerControlTaskQueue()));
}
if (stickyTaskQueueName != null) {
WorkflowPollerBalancer balancer =
new WorkflowPollerBalancer(
slotSupplier.maximumSlots(),
((PollerBehaviorAutoscaling) pollerOptions.getPollerBehavior())
.getInitialConcurrentTaskPollers());
for (AsyncWorkflowPollTask task : pollers) {
task.setPollerBalancer(balancer);
}
}
poller =
new AsyncPoller<>(
slotSupplier,
Expand Down
Loading
Loading