From bb6e99f310b4e7417f4522e7950379ccbf746ebf Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 05:13:03 -0600 Subject: [PATCH 01/10] Add durable proxy vote delivery --- .mex/events/decisions.jsonl | 2 + AGENTS.md | 4 + .../votingplugin/VotingPluginMain.java | 14 +- .../backendproxy/BackendProxyHandler.java | 19 +- .../cache/DurableVoteReceiptStore.java | 140 +++++++++++++ .../cache/ProcessedVoteCache.java | 32 ++- .../messaging/BackendProxyMessageRouter.java | 22 +- .../proxy/ReliableVoteDeliveryOutbox.java | 190 ++++++++++++++++++ .../votingplugin/proxy/VotingPluginProxy.java | 91 ++++++++- .../votingplugin/proxy/VotingPluginWire.java | 40 +++- .../BackendProxyHandlerLifecycleTest.java | 41 ++++ .../ProcessedVoteCacheDurabilityTest.java | 35 ++++ .../BackendProxyMessageRouterTest.java | 31 +++ .../proxy/ReliableVoteDeliveryOutboxTest.java | 84 ++++++++ .../tests/VotingPluginWireTest.java | 20 ++ docs/proxy-vote-delivery.md | 29 +++ 16 files changed, 775 insertions(+), 19 deletions(-) create mode 100644 .mex/events/decisions.jsonl create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java create mode 100644 docs/proxy-vote-delivery.md diff --git a/.mex/events/decisions.jsonl b/.mex/events/decisions.jsonl new file mode 100644 index 000000000..73c262b6f --- /dev/null +++ b/.mex/events/decisions.jsonl @@ -0,0 +1,2 @@ +{"timestamp":"2026-09-21T10:51:59.003Z","kind":"decision","message":"Proxy-to-backend reward votes use a capability-negotiated durable outbox and completion acknowledgement. The guarantee is at least once; HTTP retains its existing delivery path, old backends retain legacy semantics, and exactly-once reward effects remain out of scope.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java"],"cwd":".","source":"agent","status":"implemented"} +{"timestamp":"2026-09-21T11:07:28.400Z","kind":"decision","message":"Reliable backend votes persist completed vote IDs for seven days before acknowledgement. Receipt persistence failure retries without repeating effects in the active process; restart deduplication covers durable completions, while a crash during non-transactional reward effects remains at-least-once.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java"],"cwd":".","source":"agent","status":"implemented"} diff --git a/AGENTS.md b/AGENTS.md index 74472b23b..d418a35ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,6 +176,10 @@ credentials, generated JARs, dependency caches, IDE output, or unrelated formatt - Trace whether the code runs on the connector worker, proxy thread, Bukkit primary thread, or a SQL executor. - Preserve queued votes across saturation, shutdown, and restart; overflow handling must be bounded, durable when promised, and observable rather than silently dropping work. +- Proxy-to-backend guaranteed delivery is capability negotiated and at least once. Journal a reward-bearing envelope before + reporting transport acceptance, retain it until the matching backend completion acknowledgement is durable, persist + completed IDs before acknowledgement for restart-safe deduplication, and keep legacy send behavior for backends that do + not advertise the capability. - Treat scheduler units explicitly. Verify whether each delay is in ticks, milliseconds, or seconds, especially across Bukkit, Folia, BungeeCord, and Velocity adapters. - Register listeners and lifecycle wakeups before producers can publish work; startup/reload ordering must not strand already-persisted or newly-arriving operations. - Protocol-mode changes must not silently broaden legacy v1/RSA acceptance when token-only operation is configured or intended; cover downgrade behavior with tests. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 05f3eb66f..a05e47e59 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -170,7 +170,7 @@ public File getLoadedPluginJarFile() { @Getter private BackendProxyHandler backendProxyHandler; - private final ProcessedVoteCache backendProcessedVoteCache = new ProcessedVoteCache(); + private ProcessedVoteCache backendProcessedVoteCache; private BackendOrderedVoteOverflowQueue backendOrderedVoteOverflowQueue; private final AtomicReference backendPluginMessageTarget = new AtomicReference<>(); private PluginMessageHandler backendPluginMessageRelay; @@ -487,8 +487,16 @@ private synchronized BackendOrderedVoteOverflowQueue getOrCreateBackendOrderedVo return backendOrderedVoteOverflowQueue; } + private synchronized ProcessedVoteCache getOrCreateBackendProcessedVoteCache() { + if (backendProcessedVoteCache == null) { + backendProcessedVoteCache = new ProcessedVoteCache( + new File(getDataFolder(), "BackendProcessedVotes.dat").toPath()); + } + return backendProcessedVoteCache; + } + private void loadBungeeHandler() { - BackendProxyHandler candidate = new BackendProxyHandler(this, backendProcessedVoteCache, + BackendProxyHandler candidate = new BackendProxyHandler(this, getOrCreateBackendProcessedVoteCache(), getOrCreateBackendOrderedVoteOverflowQueue()); try { candidate.load(); @@ -1331,7 +1339,7 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { boolean previousRequiresPreparation = previous != null && (deferredReplacementLoad || previous.requiresPreparationForReplacement() || previous.requiresRedisRetirement()); - BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache, + BackendProxyHandler replacement = new BackendProxyHandler(this, getOrCreateBackendProcessedVoteCache(), getOrCreateBackendOrderedVoteOverflowQueue()); if (!deferredReplacementLoad) { try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 744efcf06..5b0a90fb4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -417,7 +417,7 @@ private void finishOrderedVoteDispatch(BackendOrderedVoteOverflowQueue.PendingEn boolean successful = outcome == OrderedVoteOutcome.COMPLETE; if (successful && overflowEntry != null && orderedVoteOverflow != null) { orderedVoteOverflow.acknowledgeAsync(overflowEntry, - stored -> completeOrderedVoteAcknowledgement(stored)); + stored -> completeOrderedVoteAcknowledgement(stored, envelope)); return; } synchronized (orderedVoteDispatch) { @@ -432,9 +432,10 @@ private void finishOrderedVoteDispatch(BackendOrderedVoteOverflowQueue.PendingEn if (successful) scheduleOrderedVoteDispatchLocked(); else retryOrderedVoteDispatchLocked(); } + if (successful) sendVoteDeliveryAcknowledgement(envelope); } - private void completeOrderedVoteAcknowledgement(boolean stored) { + private void completeOrderedVoteAcknowledgement(boolean stored, JsonEnvelope envelope) { synchronized (orderedVoteDispatch) { if (stored) { orderedVoteDispatchInFlight = null; @@ -450,6 +451,20 @@ private void completeOrderedVoteAcknowledgement(boolean stored) { orderedVoteDispatch.notifyAll(); if (stored) scheduleOrderedVoteDispatchLocked(); } + if (stored) sendVoteDeliveryAcknowledgement(envelope); + } + + private void sendVoteDeliveryAcknowledgement(JsonEnvelope envelope) { + if (!VotingPluginWire.requestsVoteDeliveryAcknowledgement(envelope) || globalMessageHandler == null) return; + String voteId = envelope.getFields().get(VotingPluginWire.K_VOTE_ID); + try { + UUID parsed = voteId == null || voteId.isBlank() ? null : UUID.fromString(voteId); + if (parsed == null) return; + globalMessageHandler.sendMessage(VotingPluginWire.voteDeliveryAcknowledgement( + plugin.getBungeeSettings().getServer(), parsed, envelope.getSubChannel())); + } catch (IllegalArgumentException invalidVoteId) { + plugin.debug("Unable to acknowledge proxy vote with invalid vote ID"); + } } private void completeOrderedVoteQuarantine(BackendOrderedVoteOverflowQueue.PendingEnvelope overflowEntry, diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java new file mode 100644 index 000000000..344ab29b4 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -0,0 +1,140 @@ +package com.bencodez.votingplugin.backendproxy.cache; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import com.bencodez.votingplugin.util.DurableFiles; + +/** Bounded append journal for backend vote IDs completed before acknowledgement. */ +final class DurableVoteReceiptStore { + static final long RECEIPT_TTL_MILLIS = TimeUnit.DAYS.toMillis(7); + private static final int MAX_RECEIPTS = 262144; + private static final long MAX_FILE_BYTES = 16L * 1024L * 1024L; + private static final String HEADER = "VP-VOTE-RECEIPTS-1"; + private static final ConcurrentHashMap FILE_LOCKS = new ConcurrentHashMap<>(); + + private final Path file; + private final Object fileLock; + private final LinkedHashMap receipts = new LinkedHashMap<>(); + private int journalRecords; + + DurableVoteReceiptStore(Path file) throws IOException { + this.file = file.toAbsolutePath().normalize(); + this.fileLock = FILE_LOCKS.computeIfAbsent(this.file, ignored -> new Object()); + synchronized (fileLock) { + load(); + } + } + + synchronized Map snapshot() { + cleanup(System.currentTimeMillis()); + return new LinkedHashMap<>(receipts); + } + + synchronized long complete(UUID voteId) { + if (voteId == null) return 0L; + long now = System.currentTimeMillis(); + cleanup(now); + Long current = receipts.get(voteId); + if (current != null && current > now) return current; + if (receipts.size() >= MAX_RECEIPTS) return 0L; + long expiresAt = now + RECEIPT_TTL_MILLIS; + String record = voteId + "\t" + expiresAt + '\n'; + synchronized (fileLock) { + if (!prepareAppend(record) || !append(record)) return 0L; + } + receipts.put(voteId, expiresAt); + journalRecords++; + return expiresAt; + } + + private void load() throws IOException { + if (!Files.exists(file, LinkOption.NOFOLLOW_LINKS)) return; + if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Vote receipt journal is not a regular file"); + } + if (Files.size(file) > MAX_FILE_BYTES) throw new IOException("Vote receipt journal exceeds size limit"); + String content = Files.readString(file, StandardCharsets.UTF_8); + String[] lines = content.split("\\n", -1); + if (lines.length == 0 || !HEADER.equals(lines[0])) throw new IOException("Unsupported vote receipt journal"); + long now = System.currentTimeMillis(); + for (int index = 1; index < lines.length; index++) { + if (lines[index].isBlank()) continue; + try { + String[] fields = lines[index].split("\\t", 2); + if (fields.length != 2) throw new IllegalArgumentException("Malformed receipt"); + UUID voteId = UUID.fromString(fields[0]); + long expiresAt = Long.parseLong(fields[1]); + if (expiresAt > now) receipts.put(voteId, expiresAt); + } catch (RuntimeException malformed) { + if (index == lines.length - 1 && !content.endsWith("\n")) return; + throw new IOException("Malformed vote receipt journal", malformed); + } + journalRecords++; + if (receipts.size() > MAX_RECEIPTS) throw new IOException("Vote receipt journal exceeds entry limit"); + } + } + + private boolean prepareAppend(String record) { + try { + long projected = (Files.exists(file) ? Files.size(file) : HEADER.length() + 1L) + + record.getBytes(StandardCharsets.UTF_8).length; + boolean shouldCompact = journalRecords > receipts.size() * 2 + 256; + if ((projected > MAX_FILE_BYTES || shouldCompact) && compact()) { + projected = Files.size(file) + record.getBytes(StandardCharsets.UTF_8).length; + } + return projected <= MAX_FILE_BYTES; + } catch (IOException failure) { + return false; + } + } + + private boolean append(String record) { + try { + Files.createDirectories(file.getParent()); + if (!Files.exists(file, LinkOption.NOFOLLOW_LINKS)) { + Path staged = file.resolveSibling(file.getFileName() + ".tmp"); + Files.writeString(staged, HEADER + '\n' + record, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + DurableFiles.publishStagedFile(staged, file); + } else { + Files.writeString(file, record, StandardCharsets.UTF_8, StandardOpenOption.APPEND, + StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS); + DurableFiles.forceFile(file); + } + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean compact() { + try { + StringBuilder content = new StringBuilder(HEADER).append('\n'); + for (Map.Entry receipt : receipts.entrySet()) { + content.append(receipt.getKey()).append('\t').append(receipt.getValue()).append('\n'); + } + Path staged = file.resolveSibling(file.getFileName() + ".tmp"); + Files.writeString(staged, content, StandardCharsets.UTF_8, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + DurableFiles.publishStagedFile(staged, file); + journalRecords = receipts.size(); + return true; + } catch (IOException failure) { + return false; + } + } + + private void cleanup(long now) { + receipts.entrySet().removeIf(entry -> entry.getValue() <= now); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index 5029b7e4d..5f04f0c9c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -1,5 +1,7 @@ package com.bencodez.votingplugin.backendproxy.cache; +import java.io.IOException; +import java.nio.file.Path; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; @@ -22,6 +24,7 @@ public class ProcessedVoteCache { @Getter private final ConcurrentHashMap processedVotes = new ConcurrentHashMap<>(); private final long ttlMillis; + private final DurableVoteReceiptStore durableReceipts; private final LinkedHashMap processedRedisDeliveries = new LinkedHashMap<>(); private final LinkedHashMap legacyRedisDeliveries = new LinkedHashMap<>(); private long legacyRedisDeliveryBytes; @@ -30,11 +33,29 @@ public class ProcessedVoteCache { private Object standbyRedisSubscriber; public ProcessedVoteCache() { - this(DEFAULT_TTL_MILLIS); + this(DEFAULT_TTL_MILLIS, null); } public ProcessedVoteCache(long ttlMillis) { + this(ttlMillis, null); + } + + public ProcessedVoteCache(Path receiptFile) { + this(DEFAULT_TTL_MILLIS, loadReceipts(receiptFile)); + } + + private ProcessedVoteCache(long ttlMillis, DurableVoteReceiptStore durableReceipts) { this.ttlMillis = ttlMillis; + this.durableReceipts = durableReceipts; + if (durableReceipts != null) processedVotes.putAll(durableReceipts.snapshot()); + } + + private static DurableVoteReceiptStore loadReceipts(Path receiptFile) { + try { + return new DurableVoteReceiptStore(receiptFile); + } catch (IOException failure) { + throw new IllegalStateException("Unable to load durable backend vote receipts", failure); + } } public boolean reserve(UUID voteId) { @@ -66,6 +87,15 @@ public boolean reserve(UUID voteId) { } } + /** Persists successful processing before the backend emits a delivery acknowledgement. */ + public boolean complete(UUID voteId) { + if (voteId == null || durableReceipts == null) return true; + long expiresAt = durableReceipts.complete(voteId); + if (expiresAt <= 0L) return false; + processedVotes.put(voteId, expiresAt); + return true; + } + /** Deduplicates one Redis envelope across overlapping subscribers during a validated handoff. */ public synchronized boolean reserveRedisDelivery(String deliveryId) { if (deliveryId == null || !deliveryId.matches("[0-9a-fA-F-]{36}")) return true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java index a68238669..b704a7849 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java @@ -82,6 +82,8 @@ public void register(GlobalMessageHandler messages, BungeeMethod method) { @Override public void onReceive(JsonEnvelope msg) { HashMap out = new HashMap<>(); out.put(VotingPluginWire.K_SERVER, nvl(plugin.getOptions().getServer())); + out.put(VotingPluginWire.K_VOTE_DELIVERY_ACK_VERSION, + VotingPluginWire.VOTE_DELIVERY_ACK_VERSION); String requestId = nvl(msg.getFields().get(VotingPluginWire.K_REQUEST_ID)); if (!requestId.isEmpty()) out.put(VotingPluginWire.K_REQUEST_ID, requestId); sendSubChannel(messages, VotingPluginWire.SUB_STATUS_OKAY, out); @@ -125,7 +127,12 @@ public void handleOrderedVote(JsonEnvelope msg, Consumer com } if (VotingPluginWire.SUB_VOTE.equals(subChannel) || VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel)) { try { - handleWireVote(msg); + UUID completedVoteId = handleWireVote(msg); + if (VotingPluginWire.requestsVoteDeliveryAcknowledgement(msg) + && completedVoteId != null && !processedVoteCache.complete(completedVoteId)) { + completion.accept(OrderedVoteOutcome.RETRY); + return; + } } catch (RuntimeException | Error failure) { completion.accept(OrderedVoteOutcome.QUARANTINE); throw failure; @@ -344,18 +351,18 @@ private void handleWireVoteDelayRejected(JsonEnvelope msg) { voteSite.giveWaitUntilVoteDelayRewards(user, rejected.wasOnline && user.isOnline(), true); } - private void handleWireVote(JsonEnvelope msg) { + private UUID handleWireVote(JsonEnvelope msg) { if (!validSchema(msg)) { - return; + return null; } VotingPluginWire.Vote vote = VotingPluginWire.readVote(msg); if (vote.uuid == null || vote.uuid.isEmpty()) { - return; + return null; } if (!ServiceSiteValidator.isValid(vote.service)) { plugin.getLogger().warning("Rejected proxy vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(vote.service) + "'"); - return; + return null; } plugin.debug("wire vote received from " + ServiceSiteValidator.sanitizeForLog(vote.player) + "/" @@ -368,7 +375,7 @@ private void handleWireVote(JsonEnvelope msg) { plugin.debug("Ignoring duplicate wire vote " + voteId + " for " + ServiceSiteValidator.sanitizeForLog(vote.player) + " on " + ServiceSiteValidator.sanitizeForLog(vote.service)); - return; + return voteId; } UUID javaUuid; @@ -377,7 +384,7 @@ private void handleWireVote(JsonEnvelope msg) { } catch (IllegalArgumentException e) { plugin.getLogger().warning("Invalid UUID in proxy vote: " + ServiceSiteValidator.sanitizeForLog(vote.uuid)); - return; + return voteId; } VotingPluginUser user = plugin.getVotingPluginUserManager().getVotingPluginUser(javaUuid, vote.player); votePartySync.replace(totals.getVotePartyCurrent(), totals.getVotePartyRequired()); @@ -390,6 +397,7 @@ private void handleWireVote(JsonEnvelope msg) { if (vote.service != null && !vote.service.isEmpty()) { plugin.getServerData().addServiceSite(vote.service); } + return voteId; } private boolean validSchema(JsonEnvelope msg) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java new file mode 100644 index 000000000..20e401ffd --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java @@ -0,0 +1,190 @@ +package com.bencodez.votingplugin.proxy; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.UUID; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.bencodez.votingplugin.util.DurableFiles; + +/** Durable proxy outbox for reward-bearing votes awaiting backend completion. */ +final class ReliableVoteDeliveryOutbox { + static final int MAX_ENTRIES = 4096; + private static final long MAX_FILE_BYTES = 16L * 1024L * 1024L; + private static final String HEADER = "VP-VOTE-OUTBOX-2"; + private static final String ADD = "A"; + private static final String REMOVE = "R"; + + record Entry(String server, JsonEnvelope envelope) { } + + private final Path file; + private final LinkedHashMap entries = new LinkedHashMap<>(); + private int journalRecords; + + ReliableVoteDeliveryOutbox(Path file) throws IOException { + this.file = file.toAbsolutePath().normalize(); + load(); + } + + synchronized boolean offer(String server, JsonEnvelope envelope) { + String key = key(server, envelope); + if (key == null) return false; + if (entries.containsKey(key)) return true; + if (entries.size() >= MAX_ENTRIES) return false; + String record = addRecord(server, envelope); + if (!prepareAppend(record) || !append(record)) return false; + entries.put(key, new Entry(server, envelope)); + journalRecords++; + return true; + } + + synchronized boolean acknowledge(String server, UUID voteId, String subChannel) { + if (voteId == null || server == null || subChannel == null) return false; + String key = normalized(server) + '|' + subChannel + '|' + voteId; + if (!entries.containsKey(key)) return false; + if (entries.size() == 1) { + try { + if (!DurableFiles.deleteIfExists(file) && Files.exists(file)) return false; + } catch (IOException failure) { + return false; + } + entries.clear(); + journalRecords = 0; + return true; + } + String record = REMOVE + '\t' + encode(key) + '\n'; + if (!prepareAppend(record) || !append(record)) return false; + entries.remove(key); + journalRecords++; + return true; + } + + synchronized List snapshot() { + return new ArrayList<>(entries.values()); + } + + synchronized int size() { + return entries.size(); + } + + private void load() throws IOException { + if (!Files.exists(file, LinkOption.NOFOLLOW_LINKS)) return; + if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Vote delivery outbox is not a regular file"); + } + if (Files.size(file) > MAX_FILE_BYTES) throw new IOException("Vote delivery outbox exceeds size limit"); + String content = Files.readString(file, StandardCharsets.UTF_8); + String[] lines = content.split("\\n", -1); + if (lines.length == 0 || !HEADER.equals(lines[0])) throw new IOException("Unsupported vote delivery outbox"); + for (int index = 1; index < lines.length; index++) { + if (lines[index].isBlank()) continue; + String[] parts = lines[index].split("\\t", 3); + try { + if (parts.length == 3 && ADD.equals(parts[0])) { + String server = decode(parts[1]); + JsonEnvelope envelope = JsonEnvelopeCodec.decode(decode(parts[2])); + String key = key(server, envelope); + if (key == null) throw new IllegalArgumentException("Invalid vote envelope"); + entries.put(key, new Entry(server, envelope)); + } else if (parts.length == 2 && REMOVE.equals(parts[0])) { + entries.remove(decode(parts[1])); + } else throw new IllegalArgumentException("Unknown journal record"); + } catch (RuntimeException malformed) { + // An append interrupted before force may leave only the final record + // truncated. Its caller never observed durable acceptance, so replaying + // the preceding journal is safe and keeps startup available. + if (index == lines.length - 1 && !content.endsWith("\n")) return; + throw new IOException("Malformed vote delivery outbox entry", malformed); + } + journalRecords++; + if (entries.size() > MAX_ENTRIES) throw new IOException("Vote delivery outbox exceeds entry limit"); + } + } + + private boolean prepareAppend(String record) { + try { + long projectedBytes = (Files.exists(file) ? Files.size(file) : HEADER.length() + 1L) + + record.getBytes(StandardCharsets.UTF_8).length; + boolean shouldCompact = journalRecords > entries.size() * 2 + 64; + if ((projectedBytes > MAX_FILE_BYTES || shouldCompact) && compact()) { + projectedBytes = Files.size(file) + record.getBytes(StandardCharsets.UTF_8).length; + } + return projectedBytes <= MAX_FILE_BYTES; + } catch (IOException failure) { + return false; + } + } + + private boolean append(String record) { + try { + Files.createDirectories(file.getParent()); + if (!Files.exists(file, LinkOption.NOFOLLOW_LINKS)) { + Path staged = file.resolveSibling(file.getFileName() + ".tmp"); + Files.writeString(staged, HEADER + '\n' + record, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + DurableFiles.publishStagedFile(staged, file); + } else { + Files.writeString(file, record, StandardCharsets.UTF_8, StandardOpenOption.APPEND, + StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS); + DurableFiles.forceFile(file); + } + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean compact() { + try { + StringBuilder text = new StringBuilder(HEADER).append('\n'); + for (Entry entry : entries.values()) text.append(addRecord(entry.server(), entry.envelope())); + Path staged = file.resolveSibling(file.getFileName() + ".tmp"); + Files.writeString(staged, text, StandardCharsets.UTF_8, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + DurableFiles.publishStagedFile(staged, file); + journalRecords = entries.size(); + return true; + } catch (IOException failure) { + return false; + } + } + + private static String addRecord(String server, JsonEnvelope envelope) { + return ADD + '\t' + encode(server) + '\t' + encode(JsonEnvelopeCodec.encode(envelope)) + '\n'; + } + + private static String key(String server, JsonEnvelope envelope) { + if (server == null || server.isBlank() || envelope == null) return null; + String subChannel = envelope.getSubChannel(); + if (!VotingPluginWire.SUB_VOTE.equals(subChannel) + && !VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel)) return null; + String voteId = envelope.getFields().get(VotingPluginWire.K_VOTE_ID); + try { + return normalized(server) + '|' + subChannel + '|' + UUID.fromString(voteId); + } catch (RuntimeException invalid) { + return null; + } + } + + private static String normalized(String value) { + return value.trim().toLowerCase(Locale.ROOT); + } + + private static String encode(String value) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(value.getBytes(StandardCharsets.UTF_8)); + } + + private static String decode(String value) { + return new String(Base64.getUrlDecoder().decode(value), StandardCharsets.UTF_8); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index be7053e6b..dc4460a2e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -128,6 +128,8 @@ private VoteRetryException() { private final Map liveVoteRetries = new LinkedHashMap<>(); private final Map multiProxyVoteRetries = new LinkedHashMap<>(); private final LinkedHashMap completedMultiProxyVotes = new LinkedHashMap<>(); + private final Set reliableVoteDeliveryServers = ConcurrentHashMap.newKeySet(); + private ReliableVoteDeliveryOutbox reliableVoteDeliveryOutbox; // Set only after all replacement gates have succeeded. Vote entry points are // synchronized, so no new side-effecting vote can race the handoff window. private boolean runtimeReplacementPrepared; @@ -855,10 +857,76 @@ protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelop if (handler == null) { return false; } + if (supportsReliableVoteDelivery(server)) { + ReliableVoteDeliveryOutbox outbox = reliableVoteDeliveryOutbox; + if (outbox == null || !outbox.offer(server, envelope)) { + logSevere("Unable to durably queue vote delivery for " + server); + return false; + } + try { + handler.sendMessage(server, delay, + VotingPluginWire.requestVoteDeliveryAcknowledgement(envelope)); + } catch (RuntimeException failure) { + debug("Vote delivery remains queued after the immediate send failed for " + server); + } + return true; + } handler.sendMessage(server, delay, envelope); return true; } + private boolean supportsReliableVoteDelivery(String server) { + return server != null && reliableVoteDeliveryServers.contains(server.trim().toLowerCase(Locale.ROOT)); + } + + private void updateReliableVoteDeliveryCapability(String server, JsonEnvelope message) { + if (server == null || server.isBlank() || !isServerValid(server)) return; + String key = server.trim().toLowerCase(Locale.ROOT); + if (VotingPluginWire.advertisesVoteDeliveryAcknowledgement(message)) { + reliableVoteDeliveryServers.add(key); + retryReliableVoteDeliveries(server); + } else { + reliableVoteDeliveryServers.remove(key); + } + } + + private void retryReliableVoteDeliveries() { + retryReliableVoteDeliveries(null); + } + + private void retryReliableVoteDeliveries(String onlyServer) { + ReliableVoteDeliveryOutbox outbox = reliableVoteDeliveryOutbox; + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (outbox == null || handler == null) return; + int delay = 1; + for (ReliableVoteDeliveryOutbox.Entry entry : outbox.snapshot()) { + if (onlyServer != null && !entry.server().equalsIgnoreCase(onlyServer)) continue; + if (!supportsReliableVoteDelivery(entry.server())) continue; + try { + handler.sendMessage(entry.server(), delay++, + VotingPluginWire.requestVoteDeliveryAcknowledgement(entry.envelope())); + } catch (RuntimeException failure) { + debug("Vote delivery retry remains queued for " + entry.server()); + } + } + } + + private void handleVoteDeliveryAcknowledgement(JsonEnvelope message) { + if (!VotingPluginWire.advertisesVoteDeliveryAcknowledgement(message)) return; + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + String voteId = message.getFields().getOrDefault(VotingPluginWire.K_VOTE_ID, ""); + String subChannel = message.getFields().getOrDefault(VotingPluginWire.K_VOTE_DELIVERY_SUBCHANNEL, ""); + try { + if (!supportsReliableVoteDelivery(server)) return; + ReliableVoteDeliveryOutbox outbox = reliableVoteDeliveryOutbox; + if (outbox != null && !outbox.acknowledge(server, UUID.fromString(voteId), subChannel)) { + debug("Ignored unmatched or unpersisted vote delivery acknowledgement from " + server); + } + } catch (IllegalArgumentException invalidVoteId) { + debug("Ignored vote delivery acknowledgement with invalid vote ID from " + server); + } + } + public synchronized void checkCachedVotes(String server) { int delay = 1; if (isServerValid(server)) { @@ -1668,6 +1736,12 @@ public void debug1(Throwable e) { } }; voteCacheHandler.load(); + try { + reliableVoteDeliveryOutbox = new ReliableVoteDeliveryOutbox( + new File(getDataFolderPlugin(), "ProxyVoteDeliveryOutbox.dat").toPath()); + } catch (IOException failure) { + throw new IllegalStateException("Unable to load durable proxy vote delivery outbox", failure); + } method = retainHttpForPendingDeliveries(method); nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), @@ -1864,6 +1938,7 @@ && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTim VotingPluginWire.SUB_BACKEND_STARTED)) { if (backendPlayerPresenceTracker.backendStarted(server, backendIncarnationId, backendStartedAt, presenceTimestamp, System.currentTimeMillis())) { + updateReliableVoteDeliveryCapability(server, message); discardPendingPresenceHandoffs(server); pendingBackendRecoverySnapshots.add(presenceServerKey(server)); requestBackendPresenceSnapshot(server); @@ -1888,6 +1963,7 @@ && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTim if (backendPlayerPresenceTracker.backendStopped(server, backendIncarnationId, backendStartedAt, presenceTimestamp, System.currentTimeMillis())) { discardPendingPresenceHandoffs(server); + reliableVoteDeliveryServers.remove(server.trim().toLowerCase(Locale.ROOT)); pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); } } @@ -1907,8 +1983,10 @@ public void onReceive(JsonEnvelope message) { if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_HEARTBEAT) && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, VotingPluginWire.SUB_BACKEND_HEARTBEAT)) { - backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis()); + if (backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis())) { + updateReliableVoteDeliveryCapability(server, message); + } } } }); @@ -1950,10 +2028,18 @@ && isPresenceGenerationValid(snapshot.backendIncarnationId, snapshot.backendStar globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_STATUS_OKAY) { @Override public void onReceive(JsonEnvelope message) { + updateReliableVoteDeliveryCapability(message.getFields().get(VotingPluginWire.K_SERVER), message); handleStatusOkay(message); } }); + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_VOTE_DELIVERY_ACK) { + @Override + public void onReceive(JsonEnvelope message) { + handleVoteDeliveryAcknowledgement(message); + } + }); + globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { @Override public void onReceive(JsonEnvelope message) { @@ -1976,6 +2062,7 @@ public void onReceive(JsonEnvelope message) { loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, PRESENCE_MAINTENANCE_INTERVAL_SECONDS); } + loadTaskTimer(this::retryReliableVoteDeliveries, 10L, 10L); startControlServices(); // Open the listener last: backend callbacks can immediately reach routing, // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java index 598edac8a..aa7c9cefa 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java @@ -43,6 +43,7 @@ private VotingPluginWire() { public static final String SUB_VOTE_UPDATE = "VoteUpdate"; public static final String SUB_VOTE_DELAY_REJECTED = "VoteDelayRejected"; public static final String SUB_VOTE_BROADCAST = "VoteBroadcast"; + public static final String SUB_VOTE_DELIVERY_ACK = "VoteDeliveryAck"; public static final String SUB_BUNGEE_TIME_CHANGE = "BungeeTimeChange"; public static final String SUB_STATUS = "Status"; @@ -108,6 +109,9 @@ private VotingPluginWire() { public static final String K_BUNGEE_BROADCAST = "bungeeBroadcast"; public static final String K_NUM = "num"; public static final String K_NUMBER_OF_VOTES = "numberOfVotes"; + public static final String K_VOTE_DELIVERY_ACK_VERSION = "voteDeliveryAckVersion"; + public static final String K_VOTE_DELIVERY_SUBCHANNEL = "voteDeliverySubchannel"; + public static final int VOTE_DELIVERY_ACK_VERSION = 1; /** Origin and receiving proxy names for reliable multi-proxy delivery. */ public static final String K_MULTI_PROXY_ORIGIN = "multiProxyOrigin"; public static final String K_MULTI_PROXY_RECIPIENT = "multiProxyRecipient"; @@ -202,6 +206,30 @@ public static JsonEnvelope votePartyBungee() { return base(SUB_VOTE_PARTY).build(); } + /** Requests an acknowledgement after the backend finishes an ordered vote. */ + public static JsonEnvelope requestVoteDeliveryAcknowledgement(JsonEnvelope envelope) { + JsonEnvelope.Builder builder = JsonEnvelope.builder(envelope.getSubChannel()).schema(envelope.getSchema()); + for (Map.Entry field : envelope.getFields().entrySet()) { + builder.put(field.getKey(), field.getValue()); + } + return builder.put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); + } + + public static boolean requestsVoteDeliveryAcknowledgement(JsonEnvelope envelope) { + return readInt(envelope.getFields(), K_VOTE_DELIVERY_ACK_VERSION, 0) >= VOTE_DELIVERY_ACK_VERSION; + } + + public static boolean advertisesVoteDeliveryAcknowledgement(JsonEnvelope envelope) { + return requestsVoteDeliveryAcknowledgement(envelope); + } + + public static JsonEnvelope voteDeliveryAcknowledgement(String server, UUID voteId, String voteSubchannel) { + return base(SUB_VOTE_DELIVERY_ACK).put(K_SERVER, safe(server)) + .put(K_VOTE_ID, voteId == null ? "" : voteId.toString()) + .put(K_VOTE_DELIVERY_SUBCHANNEL, safe(voteSubchannel)) + .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); + } + public static JsonEnvelope status(String server) { return base(SUB_STATUS).put(K_SERVER, safe(server)).build(); } @@ -212,12 +240,14 @@ public static JsonEnvelope status(String server, UUID requestId) { } public static JsonEnvelope statusOkay(String server) { - return base(SUB_STATUS_OKAY).put(K_SERVER, safe(server)).build(); + return base(SUB_STATUS_OKAY).put(K_SERVER, safe(server)) + .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); } public static JsonEnvelope statusOkay(String server, UUID requestId) { return base(SUB_STATUS_OKAY).put(K_SERVER, safe(server)) - .put(K_REQUEST_ID, requestId == null ? "" : requestId.toString()).build(); + .put(K_REQUEST_ID, requestId == null ? "" : requestId.toString()) + .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); } public static JsonEnvelope serverName(String server) { @@ -287,7 +317,8 @@ public static JsonEnvelope backendStarted(String server, UUID backendIncarnation long presenceTimestamp) { return base(SUB_BACKEND_STARTED).put(K_SERVER, safe(server)).put(K_BACKEND_STARTED_AT, backendStartedAt) .put(K_BACKEND_INCARNATION_ID, backendIncarnationId == null ? "" : backendIncarnationId.toString()) - .put(K_PRESENCE_TIMESTAMP, presenceTimestamp).build(); + .put(K_PRESENCE_TIMESTAMP, presenceTimestamp) + .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); } public static JsonEnvelope backendStopped(String server) { @@ -317,7 +348,8 @@ public static JsonEnvelope backendHeartbeat(String server, UUID backendIncarnati long presenceTimestamp) { return base(SUB_BACKEND_HEARTBEAT).put(K_SERVER, safe(server)).put(K_BACKEND_STARTED_AT, backendStartedAt) .put(K_BACKEND_INCARNATION_ID, backendIncarnationId == null ? "" : backendIncarnationId.toString()) - .put(K_PRESENCE_TIMESTAMP, presenceTimestamp).build(); + .put(K_PRESENCE_TIMESTAMP, presenceTimestamp) + .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); } public static JsonEnvelope presenceResyncRequest(String server, UUID requestId, long requestedAt) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 64df6c14f..3145fa538 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -20,11 +20,13 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import java.lang.reflect.Field; import java.net.ServerSocket; import java.util.ArrayDeque; +import java.util.UUID; import java.nio.file.Path; import java.nio.file.Files; import java.util.concurrent.CompletableFuture; @@ -143,6 +145,45 @@ void voteAndVoteUpdateMessagesStayOrderedAcrossAsyncAndPlatformWork() throws Exc verify(scheduler, times(3)).runTaskAsynchronously(eq(plugin), any(Runnable.class)); } + @Test + void reliableVoteIsAcknowledgedOnlyAfterOrderedCompletion() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(settings.getServer()).thenReturn("survival"); + BackendProxyHandler handler = new BackendProxyHandler(plugin); + BackendProxyMessageRouter router = mock(BackendProxyMessageRouter.class); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + setField(handler, "messageRouter", router); + setField(handler, "globalMessageHandler", messages); + handler.activateInboundMessages(); + AtomicReference dispatch = new AtomicReference<>(); + AtomicReference> completion = new AtomicReference<>(); + doAnswer(invocation -> { + dispatch.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runTaskAsynchronously(eq(plugin), any(Runnable.class)); + doAnswer(invocation -> { + completion.set(invocation.getArgument(1)); + return null; + }).when(router).handleOrderedVote(any(JsonEnvelope.class), any()); + UUID voteId = UUID.randomUUID(); + JsonEnvelope envelope = VotingPluginWire.requestVoteDeliveryAcknowledgement( + VotingPluginWire.vote("Player", UUID.randomUUID().toString(), "site", 10L, + true, true, "", voteId, false, false, 1, 1)); + + handler.dispatchIncomingAfterPublication(envelope, mock(Runnable.class)); + dispatch.get().run(); + verifyNoInteractions(messages); + completion.get().accept(OrderedVoteOutcome.COMPLETE); + + verify(messages).sendMessage(argThat(ack -> VotingPluginWire.SUB_VOTE_DELIVERY_ACK.equals(ack.getSubChannel()) + && voteId.toString().equals(ack.getFields().get(VotingPluginWire.K_VOTE_ID)) + && "survival".equals(ack.getFields().get(VotingPluginWire.K_SERVER)))); + } + @Test void orderedVoteBacklogSpillsPastBoundToDurableOverflow(@TempDir Path tempDir) throws Exception { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java new file mode 100644 index 000000000..c1f0414d9 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -0,0 +1,35 @@ +package com.bencodez.votingplugin.backendproxy.cache; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ProcessedVoteCacheDurabilityTest { + @TempDir + Path directory; + + @Test + void completedVoteRemainsDeduplicatedAfterBackendRestart() { + Path receipts = directory.resolve("receipts.dat"); + UUID voteId = UUID.randomUUID(); + ProcessedVoteCache first = new ProcessedVoteCache(receipts); + + assertTrue(first.reserve(voteId)); + assertTrue(first.complete(voteId)); + assertFalse(new ProcessedVoteCache(receipts).reserve(voteId)); + } + + @Test + void reservationWithoutCompletionDoesNotSuppressRestartRetry() { + Path receipts = directory.resolve("receipts.dat"); + UUID voteId = UUID.randomUUID(); + assertTrue(new ProcessedVoteCache(receipts).reserve(voteId)); + + assertTrue(new ProcessedVoteCache(receipts).reserve(voteId)); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java index 4bce68c6d..d4b3c265e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -26,6 +27,7 @@ import com.bencodez.advancedcore.api.user.AdvancedCoreUser; import com.bencodez.advancedcore.api.user.usercache.UserDataManager; import com.bencodez.simpleapi.scheduler.BukkitScheduler; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.votingplugin.VotingPluginMain; import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; import com.bencodez.votingplugin.backendproxy.messaging.BackendProxyMessageRouter.OrderedVoteOutcome; @@ -38,6 +40,7 @@ import com.bencodez.votingplugin.votesites.VoteSite; import com.bencodez.votingplugin.votesites.VoteSiteManager; import com.bencodez.votingplugin.config.BungeeSettings; +import com.bencodez.votingplugin.data.ServerData; class BackendProxyMessageRouterTest { @@ -192,4 +195,32 @@ void voteRewardFailureRequestsQuarantineAfterVoteIdReservation() { assertEquals(OrderedVoteOutcome.QUARANTINE, outcome.get()); } + @Test + void reliableVoteRetriesReceiptPersistenceWithoutRepeatingEffectsInProcess() { + UUID voteId = UUID.randomUUID(); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserve(voteId)).thenReturn(true, false); + when(cache.complete(voteId)).thenReturn(false, true); + when(plugin.getBungeeSettings()).thenReturn(mock(BungeeSettings.class)); + when(plugin.getVotingPluginUserManager().getVotingPluginUser(PLAYER_UUID, "Player")) + .thenReturn(user); + when(plugin.getServerData()).thenReturn(mock(ServerData.class)); + BackendProxyMessageRouter voteRouter = new BackendProxyMessageRouter(plugin, + mock(BackendPresenceManager.class), mock(BackendGlobalDataSync.class), + mock(BackendVotePartySync.class), cache); + AtomicReference outcome = new AtomicReference<>(); + JsonEnvelope vote = VotingPluginWire.requestVoteDeliveryAcknowledgement( + VotingPluginWire.vote("Player", PLAYER_UUID.toString(), "known.example", LAST_VOTE_TIME, + true, true, "", voteId, false, false, 1, 1)); + + voteRouter.handleOrderedVote(vote, outcome::set); + assertEquals(OrderedVoteOutcome.RETRY, outcome.get()); + + voteRouter.handleOrderedVote(vote, outcome::set); + assertEquals(OrderedVoteOutcome.COMPLETE, outcome.get()); + verify(cache, times(2)).complete(voteId); + verify(user, times(1)).bungeeVotePluginMessaging(any(), anyLong(), any(), anyBoolean(), anyBoolean(), + anyBoolean(), anyInt()); + } + } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java new file mode 100644 index 000000000..eab5f1a7e --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java @@ -0,0 +1,84 @@ +package com.bencodez.votingplugin.proxy; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; + +class ReliableVoteDeliveryOutboxTest { + @TempDir + Path directory; + + @Test + void persistsUntilMatchingBackendAcknowledgesCompletion() throws Exception { + Path file = directory.resolve("outbox.dat"); + UUID voteId = UUID.randomUUID(); + JsonEnvelope vote = VotingPluginWire.vote("Player", UUID.randomUUID().toString(), "site", 10L, + true, true, "", voteId, false, false, 1, 1); + ReliableVoteDeliveryOutbox first = new ReliableVoteDeliveryOutbox(file); + + assertTrue(first.offer("Survival", vote)); + assertEquals(1, first.size()); + assertTrue(Files.isRegularFile(file)); + + ReliableVoteDeliveryOutbox restarted = new ReliableVoteDeliveryOutbox(file); + assertEquals(1, restarted.size()); + assertFalse(restarted.acknowledge("creative", voteId, VotingPluginWire.SUB_VOTE)); + assertFalse(restarted.acknowledge("survival", voteId, VotingPluginWire.SUB_VOTE_ONLINE)); + assertTrue(restarted.acknowledge("survival", voteId, VotingPluginWire.SUB_VOTE)); + assertEquals(0, restarted.size()); + assertFalse(Files.exists(file)); + } + + @Test + void duplicateOfferKeepsOneStableDelivery() throws Exception { + UUID voteId = UUID.randomUUID(); + JsonEnvelope vote = VotingPluginWire.voteOnline("Player", UUID.randomUUID().toString(), "site", 10L, + true, true, "", voteId, false, false, 1, 1); + ReliableVoteDeliveryOutbox outbox = new ReliableVoteDeliveryOutbox(directory.resolve("outbox.dat")); + + assertTrue(outbox.offer("Survival", vote)); + assertTrue(outbox.offer("survival", vote)); + assertEquals(1, outbox.size()); + assertEquals(voteId.toString(), outbox.snapshot().get(0).envelope().getFields() + .get(VotingPluginWire.K_VOTE_ID)); + } + + @Test + void removalRecordSurvivesRestartWithoutDroppingOtherVotes() throws Exception { + Path file = directory.resolve("outbox.dat"); + UUID firstId = UUID.randomUUID(); + UUID secondId = UUID.randomUUID(); + ReliableVoteDeliveryOutbox outbox = new ReliableVoteDeliveryOutbox(file); + assertTrue(outbox.offer("survival", VotingPluginWire.vote("One", UUID.randomUUID().toString(), + "site", 10L, true, true, "", firstId, false, false, 1, 1))); + assertTrue(outbox.offer("survival", VotingPluginWire.vote("Two", UUID.randomUUID().toString(), + "site", 11L, true, true, "", secondId, false, false, 1, 1))); + assertTrue(outbox.acknowledge("survival", firstId, VotingPluginWire.SUB_VOTE)); + + ReliableVoteDeliveryOutbox restarted = new ReliableVoteDeliveryOutbox(file); + assertEquals(1, restarted.size()); + assertEquals(secondId.toString(), restarted.snapshot().get(0).envelope().getFields() + .get(VotingPluginWire.K_VOTE_ID)); + } + + @Test + void restartIgnoresOnlyATruncatedFinalAppend() throws Exception { + Path file = directory.resolve("outbox.dat"); + ReliableVoteDeliveryOutbox outbox = new ReliableVoteDeliveryOutbox(file); + assertTrue(outbox.offer("survival", VotingPluginWire.vote("One", UUID.randomUUID().toString(), + "site", 10L, true, true, "", UUID.randomUUID(), false, false, 1, 1))); + Files.writeString(file, "A\ttruncated", StandardOpenOption.APPEND); + + assertEquals(1, new ReliableVoteDeliveryOutbox(file).size()); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java index 7aacf8cca..196d7c686 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java @@ -221,6 +221,26 @@ public void statusRoundTripCarriesCorrelationId() { assertEquals(requestId.toString(), request.getFields().get(VotingPluginWire.K_REQUEST_ID)); assertEquals(VotingPluginWire.SUB_STATUS_OKAY, response.getSubChannel()); assertEquals(requestId.toString(), response.getFields().get(VotingPluginWire.K_REQUEST_ID)); + assertTrue(VotingPluginWire.advertisesVoteDeliveryAcknowledgement(response)); + } + + @Test + public void voteDeliveryAcknowledgementIsAdditiveAndCorrelated() { + UUID voteId = UUID.randomUUID(); + JsonEnvelope vote = VotingPluginWire.vote("Player", UUID.randomUUID().toString(), "site", 10L, + true, true, "", voteId, false, false, 1, 1); + JsonEnvelope requested = VotingPluginWire.requestVoteDeliveryAcknowledgement(vote); + + assertFalse(VotingPluginWire.requestsVoteDeliveryAcknowledgement(vote)); + assertTrue(VotingPluginWire.requestsVoteDeliveryAcknowledgement(requested)); + assertEquals(voteId.toString(), requested.getFields().get(VotingPluginWire.K_VOTE_ID)); + + JsonEnvelope acknowledgement = VotingPluginWire.voteDeliveryAcknowledgement( + "survival", voteId, VotingPluginWire.SUB_VOTE); + assertEquals("survival", acknowledgement.getFields().get(VotingPluginWire.K_SERVER)); + assertEquals(voteId.toString(), acknowledgement.getFields().get(VotingPluginWire.K_VOTE_ID)); + assertEquals(VotingPluginWire.SUB_VOTE, + acknowledgement.getFields().get(VotingPluginWire.K_VOTE_DELIVERY_SUBCHANNEL)); } @Test diff --git a/docs/proxy-vote-delivery.md b/docs/proxy-vote-delivery.md new file mode 100644 index 000000000..fbc3829c0 --- /dev/null +++ b/docs/proxy-vote-delivery.md @@ -0,0 +1,29 @@ +# Proxy vote delivery + +Reward-bearing `Vote` and `VoteOnline` messages use an additive, versioned +acknowledgement protocol when both the proxy and backend support it. The backend +advertises `voteDeliveryAckVersion` through presence or status replies. Older +backends ignore the extra fields and retain the existing transport behavior. + +Before a capable route reports acceptance to the existing vote pipeline, the +proxy writes the exact target server and envelope to +`ProxyVoteDeliveryOutbox.dat`. It retries that entry every ten seconds and after +capability discovery. The backend acknowledges only after its ordered vote lane +reports completion. A spilled message is acknowledged after removal from the +durable backend overflow queue. Before acknowledgement, the backend journals the +completed vote ID for seven days so a lost acknowledgement followed by backend +restart does not repeat normal completed processing. The proxy then durably +removes the matching server, vote ID, and subchannel entry from its outbox. + +This is an **at least once delivery guarantee**. Proxy shutdown, restart, a lost +send, or a lost acknowledgement leaves the outbox entry available for retry. +The backend vote ID cache and durable completion journal suppress ordinary and +restart-spanning duplicate retries. A backend process crash during reward side +effects, before the completion record is durable, can still lead to a repeated +attempt because reward execution and the receipt cannot be committed atomically. +Stronger exactly once reward execution would require a separate reward API and +storage design. + +HTTP keeps its existing request recovery and acknowledgement path. The generic +outbox covers plugin messaging, Redis, MQTT, MySQL, and sockets after capability +discovery. No reward data model or Bukkit vote ordering changes. From cffb629ec6ddbc3cdeff1f4b7549e91b82944968 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 05:22:23 -0600 Subject: [PATCH 02/10] Handle delivery downgrade and journal recovery --- .../cache/DurableVoteReceiptStore.java | 5 +- .../proxy/ReliableVoteDeliveryOutbox.java | 5 +- .../votingplugin/proxy/VotingPluginProxy.java | 22 +++++++-- .../ProcessedVoteCacheDurabilityTest.java | 20 ++++++++ .../proxy/ReliableVoteDeliveryOutboxTest.java | 6 ++- .../proxy/VotingPluginProxyLifecycleTest.java | 47 +++++++++++++++++-- docs/proxy-vote-delivery.md | 4 ++ 7 files changed, 98 insertions(+), 11 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java index 344ab29b4..55cefafeb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -76,7 +76,10 @@ private void load() throws IOException { long expiresAt = Long.parseLong(fields[1]); if (expiresAt > now) receipts.put(voteId, expiresAt); } catch (RuntimeException malformed) { - if (index == lines.length - 1 && !content.endsWith("\n")) return; + if (index == lines.length - 1 && !content.endsWith("\n")) { + if (!compact()) throw new IOException("Unable to repair vote receipt journal", malformed); + return; + } throw new IOException("Malformed vote receipt journal", malformed); } journalRecords++; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java index 20e401ffd..297515274 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java @@ -103,7 +103,10 @@ private void load() throws IOException { // An append interrupted before force may leave only the final record // truncated. Its caller never observed durable acceptance, so replaying // the preceding journal is safe and keeps startup available. - if (index == lines.length - 1 && !content.endsWith("\n")) return; + if (index == lines.length - 1 && !content.endsWith("\n")) { + if (!compact()) throw new IOException("Unable to repair vote delivery outbox", malformed); + return; + } throw new IOException("Malformed vote delivery outbox entry", malformed); } journalRecords++; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index dc4460a2e..f78d6f464 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -129,6 +129,7 @@ private VoteRetryException() { private final Map multiProxyVoteRetries = new LinkedHashMap<>(); private final LinkedHashMap completedMultiProxyVotes = new LinkedHashMap<>(); private final Set reliableVoteDeliveryServers = ConcurrentHashMap.newKeySet(); + private final Set legacyVoteDeliveryServers = ConcurrentHashMap.newKeySet(); private ReliableVoteDeliveryOutbox reliableVoteDeliveryOutbox; // Set only after all replacement gates have succeeded. Vote entry points are // synchronized, so no new side-effecting vote can race the handoff window. @@ -883,10 +884,13 @@ private void updateReliableVoteDeliveryCapability(String server, JsonEnvelope me if (server == null || server.isBlank() || !isServerValid(server)) return; String key = server.trim().toLowerCase(Locale.ROOT); if (VotingPluginWire.advertisesVoteDeliveryAcknowledgement(message)) { + legacyVoteDeliveryServers.remove(key); reliableVoteDeliveryServers.add(key); retryReliableVoteDeliveries(server); } else { reliableVoteDeliveryServers.remove(key); + legacyVoteDeliveryServers.add(key); + retryReliableVoteDeliveries(server); } } @@ -901,10 +905,21 @@ private void retryReliableVoteDeliveries(String onlyServer) { int delay = 1; for (ReliableVoteDeliveryOutbox.Entry entry : outbox.snapshot()) { if (onlyServer != null && !entry.server().equalsIgnoreCase(onlyServer)) continue; - if (!supportsReliableVoteDelivery(entry.server())) continue; try { - handler.sendMessage(entry.server(), delay++, - VotingPluginWire.requestVoteDeliveryAcknowledgement(entry.envelope())); + if (supportsReliableVoteDelivery(entry.server())) { + handler.sendMessage(entry.server(), delay++, + VotingPluginWire.requestVoteDeliveryAcknowledgement(entry.envelope())); + } else if (legacyVoteDeliveryServers.contains(entry.server().trim().toLowerCase(Locale.ROOT))) { + handler.sendMessage(entry.server(), delay++, entry.envelope()); + String voteId = entry.envelope().getFields().get(VotingPluginWire.K_VOTE_ID); + if (!outbox.acknowledge(entry.server(), UUID.fromString(voteId), + entry.envelope().getSubChannel())) { + debug("Legacy vote delivery was accepted but remains queued until its removal is durable for " + + entry.server()); + } + } else { + continue; + } } catch (RuntimeException failure) { debug("Vote delivery retry remains queued for " + entry.server()); } @@ -1964,6 +1979,7 @@ && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTim presenceTimestamp, System.currentTimeMillis())) { discardPendingPresenceHandoffs(server); reliableVoteDeliveryServers.remove(server.trim().toLowerCase(Locale.ROOT)); + legacyVoteDeliveryServers.remove(server.trim().toLowerCase(Locale.ROOT)); pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java index c1f0414d9..5b746e0d4 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -3,7 +3,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.UUID; import org.junit.jupiter.api.Test; @@ -32,4 +34,22 @@ void reservationWithoutCompletionDoesNotSuppressRestartRetry() { assertTrue(new ProcessedVoteCache(receipts).reserve(voteId)); } + + @Test + void repairsTruncatedTailBeforeAppendingAnotherReceipt() throws Exception { + Path receipts = directory.resolve("receipts.dat"); + UUID firstId = UUID.randomUUID(); + ProcessedVoteCache first = new ProcessedVoteCache(receipts); + assertTrue(first.reserve(firstId)); + assertTrue(first.complete(firstId)); + Files.writeString(receipts, "truncated", StandardOpenOption.APPEND); + + ProcessedVoteCache repaired = new ProcessedVoteCache(receipts); + UUID secondId = UUID.randomUUID(); + assertTrue(repaired.reserve(secondId)); + assertTrue(repaired.complete(secondId)); + ProcessedVoteCache restarted = new ProcessedVoteCache(receipts); + assertFalse(restarted.reserve(firstId)); + assertFalse(restarted.reserve(secondId)); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java index eab5f1a7e..58bd73acd 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java @@ -79,6 +79,10 @@ void restartIgnoresOnlyATruncatedFinalAppend() throws Exception { "site", 10L, true, true, "", UUID.randomUUID(), false, false, 1, 1))); Files.writeString(file, "A\ttruncated", StandardOpenOption.APPEND); - assertEquals(1, new ReliableVoteDeliveryOutbox(file).size()); + ReliableVoteDeliveryOutbox repaired = new ReliableVoteDeliveryOutbox(file); + assertEquals(1, repaired.size()); + assertTrue(repaired.offer("survival", VotingPluginWire.vote("Two", UUID.randomUUID().toString(), + "site", 11L, true, true, "", UUID.randomUUID(), false, false, 1, 1))); + assertEquals(2, new ReliableVoteDeliveryOutbox(file).size()); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index 32b68ac8e..363c9484f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -1,30 +1,67 @@ package com.bencodez.votingplugin.proxy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.nio.file.Path; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.UUID; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; -import com.bencodez.votingplugin.tests.VotingPluginProxyTestImpl; -import com.bencodez.votingplugin.proxy.control.HostedControlManager; import com.bencodez.votingplugin.proxy.control.ControlConnector; +import com.bencodez.votingplugin.proxy.control.HostedControlManager; +import com.bencodez.votingplugin.tests.VotingPluginProxyTestImpl; class VotingPluginProxyLifecycleTest { + @Test + void drainsPersistedVoteThroughLegacyPathAfterCapabilityDisappears(@TempDir Path directory) throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + GlobalMessageProxyHandler messages = mock(GlobalMessageProxyHandler.class); + UUID voteId = UUID.randomUUID(); + JsonEnvelope vote = VotingPluginWire.vote("Player", UUID.randomUUID().toString(), "site", 10L, + true, true, "", voteId, false, false, 1, 1); + ReliableVoteDeliveryOutbox outbox = new ReliableVoteDeliveryOutbox(directory.resolve("outbox.dat")); + org.junit.jupiter.api.Assertions.assertTrue(outbox.offer("survival", vote)); + Field outboxField = VotingPluginProxy.class.getDeclaredField("reliableVoteDeliveryOutbox"); + outboxField.setAccessible(true); + outboxField.set(proxy, outbox); + Field messagesField = VotingPluginProxy.class.getDeclaredField("globalMessageProxyHandler"); + messagesField.setAccessible(true); + messagesField.set(proxy, messages); + Field legacyServers = VotingPluginProxy.class.getDeclaredField("legacyVoteDeliveryServers"); + legacyServers.setAccessible(true); + @SuppressWarnings("unchecked") + Set legacy = (Set) legacyServers.get(proxy); + Method retry = VotingPluginProxy.class.getDeclaredMethod("retryReliableVoteDeliveries", String.class); + retry.setAccessible(true); + retry.invoke(proxy, "survival"); + assertEquals(1, outbox.size()); + org.mockito.Mockito.verifyNoInteractions(messages); + + legacy.add("survival"); + retry.invoke(proxy, "survival"); + + verify(messages).sendMessage("survival", 1, vote); + assertEquals(0, outbox.size()); + } @Test void retainsConnectorWhenOperationShutdownFails() throws Exception { diff --git a/docs/proxy-vote-delivery.md b/docs/proxy-vote-delivery.md index fbc3829c0..236a0e16a 100644 --- a/docs/proxy-vote-delivery.md +++ b/docs/proxy-vote-delivery.md @@ -14,6 +14,10 @@ durable backend overflow queue. Before acknowledgement, the backend journals the completed vote ID for seven days so a lost acknowledgement followed by backend restart does not repeat normal completed processing. The proxy then durably removes the matching server, vote ID, and subchannel entry from its outbox. +If a backend generation stops advertising acknowledgements, the proxy drains +already-journaled entries once through the existing legacy send path and removes +each entry only after that send is accepted. This keeps rolling downgrades from +stranding accepted votes while retaining at-least-once behavior. This is an **at least once delivery guarantee**. Proxy shutdown, restart, a lost send, or a lost acknowledgement leaves the outbox entry available for retry. From 430df879302585e7946d8d714bc27f16f18d0c9a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 05:31:05 -0600 Subject: [PATCH 03/10] Complete capability discovery and downgrade delivery --- .../votingplugin/proxy/VotingPluginProxy.java | 23 +++++++++++++--- .../proxy/VotingPluginProxyLifecycleTest.java | 27 ++++++++++++++++++- docs/proxy-vote-delivery.md | 4 ++- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index f78d6f464..bec9205a1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -198,6 +198,7 @@ public void run() { } } private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; + private static final long VOTE_DELIVERY_CAPABILITY_PROBE_SECONDS = 60L; private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); private static final int MAX_PENDING_VOTE_PARTY_REWARDS = 1024; @@ -907,10 +908,15 @@ private void retryReliableVoteDeliveries(String onlyServer) { if (onlyServer != null && !entry.server().equalsIgnoreCase(onlyServer)) continue; try { if (supportsReliableVoteDelivery(entry.server())) { - handler.sendMessage(entry.server(), delay++, + handler.sendMessage(entry.server(), delay++, VotingPluginWire.requestVoteDeliveryAcknowledgement(entry.envelope())); } else if (legacyVoteDeliveryServers.contains(entry.server().trim().toLowerCase(Locale.ROOT))) { - handler.sendMessage(entry.server(), delay++, entry.envelope()); + if (!sendProxyBroadcastEnvelopeNow(entry.server(), entry.envelope())) { + debug("Legacy vote delivery remains queued because the transport rejected it for " + + entry.server()); + continue; + } + delay++; String voteId = entry.envelope().getFields().get(VotingPluginWire.K_VOTE_ID); if (!outbox.acknowledge(entry.server(), UUID.fromString(voteId), entry.envelope().getSubChannel())) { @@ -926,6 +932,14 @@ private void retryReliableVoteDeliveries(String onlyServer) { } } + private void probeReliableVoteDeliveryCapabilities() { + if (method != BungeeMethod.PLUGINMESSAGING || globalMessageProxyHandler == null) return; + int delay = 1; + for (String server : getAllAvailableServers()) { + globalMessageProxyHandler.sendMessage(server, delay++, VotingPluginWire.status(server)); + } + } + private void handleVoteDeliveryAcknowledgement(JsonEnvelope message) { if (!VotingPluginWire.advertisesVoteDeliveryAcknowledgement(message)) return; String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); @@ -1941,10 +1955,11 @@ public void onReceive(JsonEnvelope message) { globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STARTED) { @Override public void onReceive(JsonEnvelope message) { + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); if (!method.supportsBackendPresence()) { + updateReliableVoteDeliveryCapability(server, message); return; } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); @@ -2079,6 +2094,8 @@ public void onReceive(JsonEnvelope message) { PRESENCE_MAINTENANCE_INTERVAL_SECONDS); } loadTaskTimer(this::retryReliableVoteDeliveries, 10L, 10L); + loadTaskTimer(this::probeReliableVoteDeliveryCapabilities, 1L, + VOTE_DELIVERY_CAPABILITY_PROBE_SECONDS); startControlServices(); // Open the listener last: backend callbacks can immediately reach routing, // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers. diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index 363c9484f..c68445c26 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -34,6 +34,7 @@ class VotingPluginProxyLifecycleTest { @Test void drainsPersistedVoteThroughLegacyPathAfterCapabilityDisappears(@TempDir Path directory) throws Exception { VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + proxy.setMethod(BungeeMethod.PLUGINMESSAGING); GlobalMessageProxyHandler messages = mock(GlobalMessageProxyHandler.class); UUID voteId = UUID.randomUUID(); JsonEnvelope vote = VotingPluginWire.vote("Player", UUID.randomUUID().toString(), "site", 10L, @@ -57,12 +58,36 @@ void drainsPersistedVoteThroughLegacyPathAfterCapabilityDisappears(@TempDir Path org.mockito.Mockito.verifyNoInteractions(messages); legacy.add("survival"); + proxy.setPluginMessageDeliveryResult(false); + retry.invoke(proxy, "survival"); + assertEquals(1, outbox.size()); + + proxy.setPluginMessageDeliveryResult(true); retry.invoke(proxy, "survival"); - verify(messages).sendMessage("survival", 1, vote); assertEquals(0, outbox.size()); } + @Test + void probesPluginMessagingBackendsForDeliveryCapability() throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + proxy.setMethod(BungeeMethod.PLUGINMESSAGING); + proxy.setAvailableServers("survival"); + GlobalMessageProxyHandler messages = mock(GlobalMessageProxyHandler.class); + Field messagesField = VotingPluginProxy.class.getDeclaredField("globalMessageProxyHandler"); + messagesField.setAccessible(true); + messagesField.set(proxy, messages); + Method probe = VotingPluginProxy.class.getDeclaredMethod("probeReliableVoteDeliveryCapabilities"); + probe.setAccessible(true); + + probe.invoke(proxy); + + org.mockito.ArgumentCaptor envelope = org.mockito.ArgumentCaptor.forClass(JsonEnvelope.class); + verify(messages).sendMessage(org.mockito.ArgumentMatchers.eq("survival"), + org.mockito.ArgumentMatchers.eq(1), envelope.capture()); + assertEquals(VotingPluginWire.SUB_STATUS, envelope.getValue().getSubChannel()); + } + @Test void retainsConnectorWhenOperationShutdownFails() throws Exception { VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); diff --git a/docs/proxy-vote-delivery.md b/docs/proxy-vote-delivery.md index 236a0e16a..e98354d96 100644 --- a/docs/proxy-vote-delivery.md +++ b/docs/proxy-vote-delivery.md @@ -4,6 +4,8 @@ Reward-bearing `Vote` and `VoteOnline` messages use an additive, versioned acknowledgement protocol when both the proxy and backend support it. The backend advertises `voteDeliveryAckVersion` through presence or status replies. Older backends ignore the extra fields and retain the existing transport behavior. +Plugin messaging probes each available backend every minute because that +transport does not use backend presence heartbeats. Before a capable route reports acceptance to the existing vote pipeline, the proxy writes the exact target server and envelope to @@ -16,7 +18,7 @@ restart does not repeat normal completed processing. The proxy then durably removes the matching server, vote ID, and subchannel entry from its outbox. If a backend generation stops advertising acknowledgements, the proxy drains already-journaled entries once through the existing legacy send path and removes -each entry only after that send is accepted. This keeps rolling downgrades from +each entry only after the selected transport reports acceptance. This keeps rolling downgrades from stranding accepted votes while retaining at-least-once behavior. This is an **at least once delivery guarantee**. Proxy shutdown, restart, a lost From ef14b7223fa0b5a911ce8cd224a777c02c8dab9a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 05:45:46 -0600 Subject: [PATCH 04/10] Close remaining delivery guarantee gaps --- .mex/events/decisions.jsonl | 1 + .../cache/DurableVoteReceiptStore.java | 16 ++--------- .../cache/ProcessedVoteCache.java | 12 ++++++-- .../votingplugin/proxy/VotingPluginProxy.java | 28 +++++++++++-------- .../ProcessedVoteCacheDurabilityTest.java | 13 +++++++++ .../proxy/VotingPluginProxyLifecycleTest.java | 25 +++++++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 7 +++-- docs/proxy-vote-delivery.md | 13 +++++---- 8 files changed, 80 insertions(+), 35 deletions(-) diff --git a/.mex/events/decisions.jsonl b/.mex/events/decisions.jsonl index 73c262b6f..489017c77 100644 --- a/.mex/events/decisions.jsonl +++ b/.mex/events/decisions.jsonl @@ -1,2 +1,3 @@ {"timestamp":"2026-09-21T10:51:59.003Z","kind":"decision","message":"Proxy-to-backend reward votes use a capability-negotiated durable outbox and completion acknowledgement. The guarantee is at least once; HTTP retains its existing delivery path, old backends retain legacy semantics, and exactly-once reward effects remain out of scope.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java"],"cwd":".","source":"agent","status":"implemented"} {"timestamp":"2026-09-21T11:07:28.400Z","kind":"decision","message":"Reliable backend votes persist completed vote IDs for seven days before acknowledgement. Receipt persistence failure retries without repeating effects in the active process; restart deduplication covers durable completions, while a crash during non-transactional reward effects remains at-least-once.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java"],"cwd":".","source":"agent","status":"implemented"} +{"timestamp":"2026-09-21T11:41:29.374Z","kind":"decision","message":"Supersedes the seven-day receipt decision: completion receipts have no time expiry and fail closed at the bounded journal capacity, HTTP participates in the completion-acknowledged outbox, and receipt-write failures keep an in-memory post-effect fence until persistence succeeds.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java"],"cwd":".","source":"agent","status":"implemented"} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java index 55cefafeb..4c074c736 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -10,13 +10,11 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import com.bencodez.votingplugin.util.DurableFiles; /** Bounded append journal for backend vote IDs completed before acknowledgement. */ final class DurableVoteReceiptStore { - static final long RECEIPT_TTL_MILLIS = TimeUnit.DAYS.toMillis(7); private static final int MAX_RECEIPTS = 262144; private static final long MAX_FILE_BYTES = 16L * 1024L * 1024L; private static final String HEADER = "VP-VOTE-RECEIPTS-1"; @@ -36,18 +34,15 @@ final class DurableVoteReceiptStore { } synchronized Map snapshot() { - cleanup(System.currentTimeMillis()); return new LinkedHashMap<>(receipts); } synchronized long complete(UUID voteId) { if (voteId == null) return 0L; - long now = System.currentTimeMillis(); - cleanup(now); Long current = receipts.get(voteId); - if (current != null && current > now) return current; + if (current != null) return current; if (receipts.size() >= MAX_RECEIPTS) return 0L; - long expiresAt = now + RECEIPT_TTL_MILLIS; + long expiresAt = Long.MAX_VALUE; String record = voteId + "\t" + expiresAt + '\n'; synchronized (fileLock) { if (!prepareAppend(record) || !append(record)) return 0L; @@ -66,7 +61,6 @@ private void load() throws IOException { String content = Files.readString(file, StandardCharsets.UTF_8); String[] lines = content.split("\\n", -1); if (lines.length == 0 || !HEADER.equals(lines[0])) throw new IOException("Unsupported vote receipt journal"); - long now = System.currentTimeMillis(); for (int index = 1; index < lines.length; index++) { if (lines[index].isBlank()) continue; try { @@ -74,7 +68,7 @@ private void load() throws IOException { if (fields.length != 2) throw new IllegalArgumentException("Malformed receipt"); UUID voteId = UUID.fromString(fields[0]); long expiresAt = Long.parseLong(fields[1]); - if (expiresAt > now) receipts.put(voteId, expiresAt); + receipts.put(voteId, expiresAt); } catch (RuntimeException malformed) { if (index == lines.length - 1 && !content.endsWith("\n")) { if (!compact()) throw new IOException("Unable to repair vote receipt journal", malformed); @@ -136,8 +130,4 @@ private boolean compact() { return false; } } - - private void cleanup(long now) { - receipts.entrySet().removeIf(entry -> entry.getValue() <= now); - } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index 5f04f0c9c..f0306eb24 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -23,6 +23,7 @@ public class ProcessedVoteCache { @Getter private final ConcurrentHashMap processedVotes = new ConcurrentHashMap<>(); + private final java.util.Set completedAwaitingReceipt = ConcurrentHashMap.newKeySet(); private final long ttlMillis; private final DurableVoteReceiptStore durableReceipts; private final LinkedHashMap processedRedisDeliveries = new LinkedHashMap<>(); @@ -33,17 +34,21 @@ public class ProcessedVoteCache { private Object standbyRedisSubscriber; public ProcessedVoteCache() { - this(DEFAULT_TTL_MILLIS, null); + this(DEFAULT_TTL_MILLIS, (DurableVoteReceiptStore) null); } public ProcessedVoteCache(long ttlMillis) { - this(ttlMillis, null); + this(ttlMillis, (DurableVoteReceiptStore) null); } public ProcessedVoteCache(Path receiptFile) { this(DEFAULT_TTL_MILLIS, loadReceipts(receiptFile)); } + ProcessedVoteCache(long ttlMillis, Path receiptFile) { + this(ttlMillis, loadReceipts(receiptFile)); + } + private ProcessedVoteCache(long ttlMillis, DurableVoteReceiptStore durableReceipts) { this.ttlMillis = ttlMillis; this.durableReceipts = durableReceipts; @@ -67,6 +72,7 @@ public boolean reserve(UUID voteId) { long expiresAt = now + ttlMillis; while (true) { + if (completedAwaitingReceipt.contains(voteId)) return false; Long currentExpiry = processedVotes.get(voteId); if (currentExpiry == null) { if (processedVotes.putIfAbsent(voteId, expiresAt) == null) { @@ -90,9 +96,11 @@ public boolean reserve(UUID voteId) { /** Persists successful processing before the backend emits a delivery acknowledgement. */ public boolean complete(UUID voteId) { if (voteId == null || durableReceipts == null) return true; + completedAwaitingReceipt.add(voteId); long expiresAt = durableReceipts.complete(voteId); if (expiresAt <= 0L) return false; processedVotes.put(voteId, expiresAt); + completedAwaitingReceipt.remove(voteId); return true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index bec9205a1..f5c034d66 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -852,27 +852,31 @@ protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelop protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { - if (method == BungeeMethod.HTTP) { - return sendHttpEnvelopeWithRecovery(server, envelope, cachedVote); - } - GlobalMessageProxyHandler handler = globalMessageProxyHandler; - if (handler == null) { - return false; - } if (supportsReliableVoteDelivery(server)) { ReliableVoteDeliveryOutbox outbox = reliableVoteDeliveryOutbox; if (outbox == null || !outbox.offer(server, envelope)) { logSevere("Unable to durably queue vote delivery for " + server); return false; } - try { - handler.sendMessage(server, delay, - VotingPluginWire.requestVoteDeliveryAcknowledgement(envelope)); - } catch (RuntimeException failure) { - debug("Vote delivery remains queued after the immediate send failed for " + server); + JsonEnvelope requested = VotingPluginWire.requestVoteDeliveryAcknowledgement(envelope); + if (method == BungeeMethod.HTTP) sendGenericHttpEnvelope(server, requested); + else { + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler != null) { + try { + handler.sendMessage(server, delay, requested); + } catch (RuntimeException failure) { + debug("Vote delivery remains queued after the immediate send failed for " + server); + } + } } return true; } + if (method == BungeeMethod.HTTP) { + return sendHttpEnvelopeWithRecovery(server, envelope, cachedVote); + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) return false; handler.sendMessage(server, delay, envelope); return true; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java index 5b746e0d4..30de45eb5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -52,4 +52,17 @@ void repairsTruncatedTailBeforeAppendingAnotherReceipt() throws Exception { assertFalse(restarted.reserve(firstId)); assertFalse(restarted.reserve(secondId)); } + + @Test + void receiptFailureFencesCompletedVotePastReservationExpiry() throws Exception { + Path unusableParent = directory.resolve("not-a-directory"); + Files.writeString(unusableParent, "file"); + UUID voteId = UUID.randomUUID(); + ProcessedVoteCache cache = new ProcessedVoteCache(1L, unusableParent.resolve("receipts.dat")); + + assertTrue(cache.reserve(voteId)); + assertFalse(cache.complete(voteId)); + Thread.sleep(5L); + assertFalse(cache.reserve(voteId)); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index c68445c26..2818e4395 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -88,6 +88,31 @@ void probesPluginMessagingBackendsForDeliveryCapability() throws Exception { assertEquals(VotingPluginWire.SUB_STATUS, envelope.getValue().getSubChannel()); } + @Test + void capableHttpVoteRemainsInOutboxWhenImmediateTransportSendIsRejected(@TempDir Path directory) + throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + proxy.setMethod(BungeeMethod.HTTP); + proxy.setVoteEnvelopeDeliveryResult(false); + ReliableVoteDeliveryOutbox outbox = new ReliableVoteDeliveryOutbox(directory.resolve("outbox.dat")); + Field outboxField = VotingPluginProxy.class.getDeclaredField("reliableVoteDeliveryOutbox"); + outboxField.setAccessible(true); + outboxField.set(proxy, outbox); + Field reliableServers = VotingPluginProxy.class.getDeclaredField("reliableVoteDeliveryServers"); + reliableServers.setAccessible(true); + @SuppressWarnings("unchecked") + Set reliable = (Set) reliableServers.get(proxy); + reliable.add("survival"); + JsonEnvelope vote = VotingPluginWire.vote("Player", UUID.randomUUID().toString(), "site", 10L, + true, true, "", UUID.randomUUID(), false, false, 1, 1); + + org.junit.jupiter.api.Assertions.assertTrue(proxy.sendVoteEnvelopeAcceptedForTest("survival", 1, vote)); + + assertEquals(1, outbox.size()); + org.junit.jupiter.api.Assertions.assertTrue( + VotingPluginWire.requestsVoteDeliveryAcknowledgement(proxy.getLastVoteEnvelope())); + } + @Test void retainsConnectorWhenOperationShutdownFails() throws Exception { VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 0f7e362ad..e5c4881c9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -379,12 +379,13 @@ public boolean sendProxyBroadcastImmediately(String server, JsonEnvelope envelop @Override protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { lastVoteEnvelope = envelope; - if (getMethod() == com.bencodez.votingplugin.proxy.BungeeMethod.HTTP) { - return voteEnvelopeDeliveryResult; - } return super.sendVoteEnvelopeAccepted(server, delay, envelope); } + public boolean sendVoteEnvelopeAcceptedForTest(String server, int delay, JsonEnvelope envelope) { + return sendVoteEnvelopeAccepted(server, delay, envelope); + } + public void setVoteEnvelopeDeliveryResult(boolean voteEnvelopeDeliveryResult) { this.voteEnvelopeDeliveryResult = voteEnvelopeDeliveryResult; } diff --git a/docs/proxy-vote-delivery.md b/docs/proxy-vote-delivery.md index e98354d96..2987e4773 100644 --- a/docs/proxy-vote-delivery.md +++ b/docs/proxy-vote-delivery.md @@ -13,8 +13,10 @@ proxy writes the exact target server and envelope to capability discovery. The backend acknowledges only after its ordered vote lane reports completion. A spilled message is acknowledged after removal from the durable backend overflow queue. Before acknowledgement, the backend journals the -completed vote ID for seven days so a lost acknowledgement followed by backend -restart does not repeat normal completed processing. The proxy then durably +completed vote ID without time expiry so a delayed or lost acknowledgement +followed by backend restart does not repeat normal completed processing. The +bounded receipt journal fails closed at its capacity instead of evicting an ID +that may still have a proxy outbox entry. The proxy then durably removes the matching server, vote ID, and subchannel entry from its outbox. If a backend generation stops advertising acknowledgements, the proxy drains already-journaled entries once through the existing legacy send path and removes @@ -30,6 +32,7 @@ attempt because reward execution and the receipt cannot be committed atomically. Stronger exactly once reward execution would require a separate reward API and storage design. -HTTP keeps its existing request recovery and acknowledgement path. The generic -outbox covers plugin messaging, Redis, MQTT, MySQL, and sockets after capability -discovery. No reward data model or Bukkit vote ordering changes. +HTTP retains its existing request recovery and also uses the completion outbox +after capability discovery. The generic outbox covers plugin messaging, Redis, +MQTT, MySQL, sockets, and HTTP. No reward data model or Bukkit vote ordering +changes. From 63be9731da9377f02306298e61b09baf98d2b68e Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 05:59:56 -0600 Subject: [PATCH 05/10] Distinguish completed and quarantined vote deliveries --- .../cache/DurableVoteReceiptStore.java | 9 +++-- .../cache/ProcessedVoteCache.java | 20 +++++++++-- .../messaging/BackendProxyMessageRouter.java | 19 ++++++++--- .../ProcessedVoteCacheDurabilityTest.java | 12 +++++++ .../BackendProxyMessageRouterTest.java | 34 ++++++++++++++++++- 5 files changed, 81 insertions(+), 13 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java index 4c074c736..be9e281cf 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -61,7 +61,9 @@ private void load() throws IOException { String content = Files.readString(file, StandardCharsets.UTF_8); String[] lines = content.split("\\n", -1); if (lines.length == 0 || !HEADER.equals(lines[0])) throw new IOException("Unsupported vote receipt journal"); - for (int index = 1; index < lines.length; index++) { + boolean unterminatedTail = !content.endsWith("\n"); + int completeLineLimit = unterminatedTail ? lines.length - 1 : lines.length; + for (int index = 1; index < completeLineLimit; index++) { if (lines[index].isBlank()) continue; try { String[] fields = lines[index].split("\\t", 2); @@ -70,15 +72,12 @@ private void load() throws IOException { long expiresAt = Long.parseLong(fields[1]); receipts.put(voteId, expiresAt); } catch (RuntimeException malformed) { - if (index == lines.length - 1 && !content.endsWith("\n")) { - if (!compact()) throw new IOException("Unable to repair vote receipt journal", malformed); - return; - } throw new IOException("Malformed vote receipt journal", malformed); } journalRecords++; if (receipts.size() > MAX_RECEIPTS) throw new IOException("Vote receipt journal exceeds entry limit"); } + if (unterminatedTail && !compact()) throw new IOException("Unable to repair vote receipt journal"); } private boolean prepareAppend(String record) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index f0306eb24..69a54f668 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -23,6 +23,7 @@ public class ProcessedVoteCache { @Getter private final ConcurrentHashMap processedVotes = new ConcurrentHashMap<>(); + private final java.util.Set completedVotes = ConcurrentHashMap.newKeySet(); private final java.util.Set completedAwaitingReceipt = ConcurrentHashMap.newKeySet(); private final long ttlMillis; private final DurableVoteReceiptStore durableReceipts; @@ -52,7 +53,11 @@ public ProcessedVoteCache(Path receiptFile) { private ProcessedVoteCache(long ttlMillis, DurableVoteReceiptStore durableReceipts) { this.ttlMillis = ttlMillis; this.durableReceipts = durableReceipts; - if (durableReceipts != null) processedVotes.putAll(durableReceipts.snapshot()); + if (durableReceipts != null) { + Map receipts = durableReceipts.snapshot(); + processedVotes.putAll(receipts); + completedVotes.addAll(receipts.keySet()); + } } private static DurableVoteReceiptStore loadReceipts(Path receiptFile) { @@ -95,15 +100,26 @@ public boolean reserve(UUID voteId) { /** Persists successful processing before the backend emits a delivery acknowledgement. */ public boolean complete(UUID voteId) { - if (voteId == null || durableReceipts == null) return true; + if (voteId == null) return true; completedAwaitingReceipt.add(voteId); + if (durableReceipts == null) { + completedVotes.add(voteId); + completedAwaitingReceipt.remove(voteId); + return true; + } long expiresAt = durableReceipts.complete(voteId); if (expiresAt <= 0L) return false; processedVotes.put(voteId, expiresAt); + completedVotes.add(voteId); completedAwaitingReceipt.remove(voteId); return true; } + /** Returns whether vote effects completed, including a receipt append awaiting retry. */ + public boolean hasCompletedEffects(UUID voteId) { + return voteId != null && (completedVotes.contains(voteId) || completedAwaitingReceipt.contains(voteId)); + } + /** Deduplicates one Redis envelope across overlapping subscribers during a validated handoff. */ public synchronized boolean reserveRedisDelivery(String deliveryId) { if (deliveryId == null || !deliveryId.matches("[0-9a-fA-F-]{36}")) return true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java index b704a7849..5823e3331 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java @@ -127,7 +127,13 @@ public void handleOrderedVote(JsonEnvelope msg, Consumer com } if (VotingPluginWire.SUB_VOTE.equals(subChannel) || VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel)) { try { - UUID completedVoteId = handleWireVote(msg); + WireVoteResult result = handleWireVote(msg); + if (VotingPluginWire.requestsVoteDeliveryAcknowledgement(msg) + && (result == null || !result.effectsComplete())) { + completion.accept(OrderedVoteOutcome.QUARANTINE); + return; + } + UUID completedVoteId = result == null ? null : result.voteId(); if (VotingPluginWire.requestsVoteDeliveryAcknowledgement(msg) && completedVoteId != null && !processedVoteCache.complete(completedVoteId)) { completion.accept(OrderedVoteOutcome.RETRY); @@ -351,7 +357,7 @@ private void handleWireVoteDelayRejected(JsonEnvelope msg) { voteSite.giveWaitUntilVoteDelayRewards(user, rejected.wasOnline && user.isOnline(), true); } - private UUID handleWireVote(JsonEnvelope msg) { + private WireVoteResult handleWireVote(JsonEnvelope msg) { if (!validSchema(msg)) { return null; } @@ -375,7 +381,7 @@ private UUID handleWireVote(JsonEnvelope msg) { plugin.debug("Ignoring duplicate wire vote " + voteId + " for " + ServiceSiteValidator.sanitizeForLog(vote.player) + " on " + ServiceSiteValidator.sanitizeForLog(vote.service)); - return voteId; + return new WireVoteResult(voteId, processedVoteCache.hasCompletedEffects(voteId)); } UUID javaUuid; @@ -384,7 +390,7 @@ private UUID handleWireVote(JsonEnvelope msg) { } catch (IllegalArgumentException e) { plugin.getLogger().warning("Invalid UUID in proxy vote: " + ServiceSiteValidator.sanitizeForLog(vote.uuid)); - return voteId; + return new WireVoteResult(voteId, false); } VotingPluginUser user = plugin.getVotingPluginUserManager().getVotingPluginUser(javaUuid, vote.player); votePartySync.replace(totals.getVotePartyCurrent(), totals.getVotePartyRequired()); @@ -397,7 +403,10 @@ private UUID handleWireVote(JsonEnvelope msg) { if (vote.service != null && !vote.service.isEmpty()) { plugin.getServerData().addServiceSite(vote.service); } - return voteId; + return new WireVoteResult(voteId, true); + } + + private record WireVoteResult(UUID voteId, boolean effectsComplete) { } private boolean validSchema(JsonEnvelope msg) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java index 30de45eb5..a3b87d77b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -53,6 +53,18 @@ void repairsTruncatedTailBeforeAppendingAnotherReceipt() throws Exception { assertFalse(restarted.reserve(secondId)); } + @Test + void discardsParseableUnterminatedReceiptTail() throws Exception { + Path receipts = directory.resolve("receipts.dat"); + UUID incompleteId = UUID.randomUUID(); + Files.writeString(receipts, "VP-VOTE-RECEIPTS-1\n" + incompleteId + "\t9"); + + ProcessedVoteCache repaired = new ProcessedVoteCache(receipts); + + assertTrue(repaired.reserve(incompleteId)); + assertTrue(Files.readString(receipts).endsWith("\n")); + } + @Test void receiptFailureFencesCompletedVotePastReservationExpiry() throws Exception { Path unusableParent = directory.resolve("not-a-directory"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java index d4b3c265e..bc1ce47aa 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java @@ -179,7 +179,7 @@ void voteUpdateFailureAfterOfflineEffectRequestsQuarantine() { void voteRewardFailureRequestsQuarantineAfterVoteIdReservation() { UUID voteId = UUID.randomUUID(); ProcessedVoteCache cache = mock(ProcessedVoteCache.class); - when(cache.reserve(voteId)).thenReturn(true); + when(cache.reserve(voteId)).thenReturn(true, false); when(plugin.getBungeeSettings()).thenReturn(mock(BungeeSettings.class)); when(plugin.getVotingPluginUserManager().getVotingPluginUser(PLAYER_UUID, "Player")) .thenReturn(user); @@ -193,6 +193,15 @@ void voteRewardFailureRequestsQuarantineAfterVoteIdReservation() { VotingPluginWire.vote("Player", PLAYER_UUID.toString(), "known.example", LAST_VOTE_TIME, true, true, "", voteId, false, false, 1, 1), outcome::set)); assertEquals(OrderedVoteOutcome.QUARANTINE, outcome.get()); + + outcome.set(null); + voteRouter.handleOrderedVote( + VotingPluginWire.requestVoteDeliveryAcknowledgement( + VotingPluginWire.vote("Player", PLAYER_UUID.toString(), "known.example", LAST_VOTE_TIME, + true, true, "", voteId, false, false, 1, 1)), + outcome::set); + assertEquals(OrderedVoteOutcome.QUARANTINE, outcome.get()); + verify(cache, never()).complete(voteId); } @Test @@ -201,6 +210,7 @@ void reliableVoteRetriesReceiptPersistenceWithoutRepeatingEffectsInProcess() { ProcessedVoteCache cache = mock(ProcessedVoteCache.class); when(cache.reserve(voteId)).thenReturn(true, false); when(cache.complete(voteId)).thenReturn(false, true); + when(cache.hasCompletedEffects(voteId)).thenReturn(true); when(plugin.getBungeeSettings()).thenReturn(mock(BungeeSettings.class)); when(plugin.getVotingPluginUserManager().getVotingPluginUser(PLAYER_UUID, "Player")) .thenReturn(user); @@ -223,4 +233,26 @@ void reliableVoteRetriesReceiptPersistenceWithoutRepeatingEffectsInProcess() { anyBoolean(), anyInt()); } + @Test + void malformedReliableVoteIsQuarantinedWithoutCompletionReceipt() { + UUID voteId = UUID.randomUUID(); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserve(voteId)).thenReturn(true); + when(plugin.getBungeeSettings()).thenReturn(mock(BungeeSettings.class)); + BackendProxyMessageRouter voteRouter = new BackendProxyMessageRouter(plugin, + mock(BackendPresenceManager.class), mock(BackendGlobalDataSync.class), + mock(BackendVotePartySync.class), cache); + AtomicReference outcome = new AtomicReference<>(); + JsonEnvelope vote = VotingPluginWire.requestVoteDeliveryAcknowledgement( + VotingPluginWire.vote("Player", "invalid-uuid", "known.example", LAST_VOTE_TIME, + true, true, "", voteId, false, false, 1, 1)); + + voteRouter.handleOrderedVote(vote, outcome::set); + + assertEquals(OrderedVoteOutcome.QUARANTINE, outcome.get()); + verify(cache, never()).complete(voteId); + verify(user, never()).bungeeVotePluginMessaging(any(), anyLong(), any(), anyBoolean(), anyBoolean(), + anyBoolean(), anyInt()); + } + } From 383ebfc992f22b8cc42850beb33066f543908155 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 06:18:31 -0600 Subject: [PATCH 06/10] Add durable vote receipt retirement --- .mex/events/decisions.jsonl | 1 + AGENTS.md | 4 +- .../BackendOrderedVoteOverflowQueue.java | 3 +- .../backendproxy/BackendProxyHandler.java | 8 ++- .../cache/DurableVoteReceiptStore.java | 33 ++++++++- .../cache/ProcessedVoteCache.java | 9 +++ .../messaging/BackendProxyMessageRouter.java | 32 +++++++++ .../proxy/ReliableVoteDeliveryOutbox.java | 72 +++++++++++++++---- .../votingplugin/proxy/VotingPluginProxy.java | 45 +++++++++++- .../votingplugin/proxy/VotingPluginWire.java | 21 +++++- .../ProcessedVoteCacheDurabilityTest.java | 13 ++++ .../BackendProxyMessageRouterTest.java | 30 ++++++++ .../proxy/ReliableVoteDeliveryOutboxTest.java | 14 ++-- .../proxy/VotingPluginProxyLifecycleTest.java | 49 +++++++++++++ .../tests/VotingPluginWireTest.java | 8 +++ docs/proxy-vote-delivery.md | 11 ++- 16 files changed, 319 insertions(+), 34 deletions(-) diff --git a/.mex/events/decisions.jsonl b/.mex/events/decisions.jsonl index 489017c77..a169bb1a3 100644 --- a/.mex/events/decisions.jsonl +++ b/.mex/events/decisions.jsonl @@ -1,3 +1,4 @@ {"timestamp":"2026-09-21T10:51:59.003Z","kind":"decision","message":"Proxy-to-backend reward votes use a capability-negotiated durable outbox and completion acknowledgement. The guarantee is at least once; HTTP retains its existing delivery path, old backends retain legacy semantics, and exactly-once reward effects remain out of scope.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java"],"cwd":".","source":"agent","status":"implemented"} {"timestamp":"2026-09-21T11:07:28.400Z","kind":"decision","message":"Reliable backend votes persist completed vote IDs for seven days before acknowledgement. Receipt persistence failure retries without repeating effects in the active process; restart deduplication covers durable completions, while a crash during non-transactional reward effects remains at-least-once.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java"],"cwd":".","source":"agent","status":"implemented"} {"timestamp":"2026-09-21T11:41:29.374Z","kind":"decision","message":"Supersedes the seven-day receipt decision: completion receipts have no time expiry and fail closed at the bounded journal capacity, HTTP participates in the completion-acknowledged outbox, and receipt-write failures keep an in-memory post-effect fence until persistence succeeds.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java"],"cwd":".","source":"agent","status":"implemented"} +{"timestamp":"2026-09-21T12:13:09.371Z","kind":"decision","message":"Supersedes indefinite unreclaimed completion receipts: reliable vote delivery uses a durable two-phase receipt-release handshake. After the proxy durably records backend completion, it retries a release marker until the backend durably removes the receipt and acknowledges retirement; only then does the proxy remove the marker.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java"],"cwd":".","source":"agent","status":"implemented"} diff --git a/AGENTS.md b/AGENTS.md index d418a35ab..a4861995f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -178,8 +178,8 @@ credentials, generated JARs, dependency caches, IDE output, or unrelated formatt - Preserve queued votes across saturation, shutdown, and restart; overflow handling must be bounded, durable when promised, and observable rather than silently dropping work. - Proxy-to-backend guaranteed delivery is capability negotiated and at least once. Journal a reward-bearing envelope before reporting transport acceptance, retain it until the matching backend completion acknowledgement is durable, persist - completed IDs before acknowledgement for restart-safe deduplication, and keep legacy send behavior for backends that do - not advertise the capability. + completed IDs before acknowledgement for restart-safe deduplication, retire receipts only through the durable + proxy-confirmed release handshake, and keep legacy send behavior for backends that do not advertise the capability. - Treat scheduler units explicitly. Verify whether each delay is in ticks, milliseconds, or seconds, especially across Bukkit, Folia, BungeeCord, and Velocity adapters. - Register listeners and lifecycle wakeups before producers can publish work; startup/reload ordering must not strand already-persisted or newly-arriving operations. - Protocol-mode changes must not silently broaden legacy v1/RSA acceptance when token-only operation is configured or intended; cover downgrade behavior with tests. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendOrderedVoteOverflowQueue.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendOrderedVoteOverflowQueue.java index e67b680d2..a457ffe47 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendOrderedVoteOverflowQueue.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendOrderedVoteOverflowQueue.java @@ -431,7 +431,8 @@ private static boolean isOrderedVoteMessage(JsonEnvelope envelope) { String subChannel = envelope.getSubChannel(); return VotingPluginWire.SUB_VOTE.equals(subChannel) || VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel) - || VotingPluginWire.SUB_VOTE_UPDATE.equals(subChannel); + || VotingPluginWire.SUB_VOTE_UPDATE.equals(subChannel) + || VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE.equals(subChannel); } private void requestPersistenceLocked() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 5b0a90fb4..ce2efb6aa 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -229,7 +229,8 @@ private boolean isOrderedVoteMessage(JsonEnvelope envelope) { String subChannel = envelope.getSubChannel(); return VotingPluginWire.SUB_VOTE.equals(subChannel) || VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel) - || VotingPluginWire.SUB_VOTE_UPDATE.equals(subChannel); + || VotingPluginWire.SUB_VOTE_UPDATE.equals(subChannel) + || VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE.equals(subChannel); } private void dispatchOrderedVote(JsonEnvelope envelope, Runnable ignoredLocalDispatch) { @@ -455,7 +456,10 @@ private void completeOrderedVoteAcknowledgement(boolean stored, JsonEnvelope env } private void sendVoteDeliveryAcknowledgement(JsonEnvelope envelope) { - if (!VotingPluginWire.requestsVoteDeliveryAcknowledgement(envelope) || globalMessageHandler == null) return; + if ((!VotingPluginWire.SUB_VOTE.equals(envelope.getSubChannel()) + && !VotingPluginWire.SUB_VOTE_ONLINE.equals(envelope.getSubChannel())) + || !VotingPluginWire.requestsVoteDeliveryAcknowledgement(envelope) + || globalMessageHandler == null) return; String voteId = envelope.getFields().get(VotingPluginWire.K_VOTE_ID); try { UUID parsed = voteId == null || voteId.isBlank() ? null : UUID.fromString(voteId); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java index be9e281cf..e1e9b295a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -18,6 +18,7 @@ final class DurableVoteReceiptStore { private static final int MAX_RECEIPTS = 262144; private static final long MAX_FILE_BYTES = 16L * 1024L * 1024L; private static final String HEADER = "VP-VOTE-RECEIPTS-1"; + private static final String RELEASE = "R"; private static final ConcurrentHashMap FILE_LOCKS = new ConcurrentHashMap<>(); private final Path file; @@ -52,6 +53,29 @@ synchronized long complete(UUID voteId) { return expiresAt; } + synchronized boolean release(UUID voteId) { + if (voteId == null || !receipts.containsKey(voteId)) return true; + if (receipts.size() == 1) { + synchronized (fileLock) { + try { + if (!DurableFiles.deleteIfExists(file) && Files.exists(file)) return false; + } catch (IOException failure) { + return false; + } + } + receipts.clear(); + journalRecords = 0; + return true; + } + String record = RELEASE + '\t' + voteId + '\n'; + synchronized (fileLock) { + if (!prepareAppend(record) || !append(record)) return false; + } + receipts.remove(voteId); + journalRecords++; + return true; + } + private void load() throws IOException { if (!Files.exists(file, LinkOption.NOFOLLOW_LINKS)) return; if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { @@ -68,9 +92,12 @@ private void load() throws IOException { try { String[] fields = lines[index].split("\\t", 2); if (fields.length != 2) throw new IllegalArgumentException("Malformed receipt"); - UUID voteId = UUID.fromString(fields[0]); - long expiresAt = Long.parseLong(fields[1]); - receipts.put(voteId, expiresAt); + if (RELEASE.equals(fields[0])) receipts.remove(UUID.fromString(fields[1])); + else { + UUID voteId = UUID.fromString(fields[0]); + long expiresAt = Long.parseLong(fields[1]); + receipts.put(voteId, expiresAt); + } } catch (RuntimeException malformed) { throw new IOException("Malformed vote receipt journal", malformed); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index 69a54f668..1ba5e948d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -120,6 +120,15 @@ public boolean hasCompletedEffects(UUID voteId) { return voteId != null && (completedVotes.contains(voteId) || completedAwaitingReceipt.contains(voteId)); } + /** Durably retires a completed receipt after the proxy confirms outbox removal. */ + public boolean releaseCompletedReceipt(UUID voteId) { + if (voteId == null) return false; + if (durableReceipts != null && !durableReceipts.release(voteId)) return false; + completedVotes.remove(voteId); + processedVotes.remove(voteId); + return true; + } + /** Deduplicates one Redis envelope across overlapping subscribers during a validated handoff. */ public synchronized boolean reserveRedisDelivery(String deliveryId) { if (deliveryId == null || !deliveryId.matches("[0-9a-fA-F-]{36}")) return true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java index 5823e3331..84e750b0a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java @@ -35,6 +35,7 @@ public enum OrderedVoteOutcome { COMPLETE, RETRY, QUARANTINE } private final BackendGlobalDataSync globalDataSync; private final BackendVotePartySync votePartySync; private final ProcessedVoteCache processedVoteCache; + private GlobalMessageHandler messages; public BackendProxyMessageRouter(VotingPluginMain plugin, BackendPresenceManager presenceManager, BackendGlobalDataSync globalDataSync, BackendVotePartySync votePartySync, @@ -47,6 +48,7 @@ public BackendProxyMessageRouter(VotingPluginMain plugin, BackendPresenceManager } public void register(GlobalMessageHandler messages, BungeeMethod method) { + this.messages = messages; messages.addListener(new GlobalMessageListener(VotingPluginWire.SUB_VOTE) { @Override public void onReceive(JsonEnvelope msg) { handleWireVote(msg); } }); @@ -56,6 +58,9 @@ public void register(GlobalMessageHandler messages, BungeeMethod method) { messages.addListener(new GlobalMessageListener(VotingPluginWire.SUB_VOTE_DELAY_REJECTED) { @Override public void onReceive(JsonEnvelope msg) { handleWireVoteDelayRejected(msg); } }); + messages.addListener(new GlobalMessageListener(VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE) { + @Override public void onReceive(JsonEnvelope msg) { handleVoteDeliveryReceiptRelease(messages, msg); } + }); messages.addListener(new GlobalMessageListener(VotingPluginWire.SUB_CONTROL_ENROLLMENT_RESULT) { @Override public void onReceive(JsonEnvelope msg) { plugin.handleBackendControlEnrollmentResult(msg); } }); @@ -121,6 +126,10 @@ void handleVoteUpdate(JsonEnvelope msg) { public void handleOrderedVote(JsonEnvelope msg, Consumer completion) { if (completion == null) throw new IllegalArgumentException("Ordered vote completion is required"); String subChannel = msg.getSubChannel(); + if (VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE.equals(subChannel)) { + completion.accept(handleVoteDeliveryReceiptRelease(messages, msg)); + return; + } if (VotingPluginWire.SUB_VOTE_UPDATE.equals(subChannel)) { handleVoteUpdateWithOutcome(msg, completion); return; @@ -418,6 +427,29 @@ private boolean validSchema(JsonEnvelope msg) { return false; } + private OrderedVoteOutcome handleVoteDeliveryReceiptRelease(GlobalMessageHandler messages, JsonEnvelope msg) { + if (messages == null || !VotingPluginWire.requestsVoteDeliveryAcknowledgement(msg)) { + return OrderedVoteOutcome.QUARANTINE; + } + String server = nvl(msg.getFields().get(VotingPluginWire.K_SERVER)); + if (!plugin.getOptions().getServer().equalsIgnoreCase(server)) return OrderedVoteOutcome.QUARANTINE; + String subChannel = nvl(msg.getFields().get(VotingPluginWire.K_VOTE_DELIVERY_SUBCHANNEL)); + if (!VotingPluginWire.SUB_VOTE.equals(subChannel) + && !VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel)) return OrderedVoteOutcome.QUARANTINE; + try { + UUID voteId = UUID.fromString(nvl(msg.getFields().get(VotingPluginWire.K_VOTE_ID))); + if (processedVoteCache.releaseCompletedReceipt(voteId)) { + messages.sendMessage(VotingPluginWire.voteDeliveryReceiptReleaseAcknowledgement( + plugin.getOptions().getServer(), voteId, subChannel)); + return OrderedVoteOutcome.COMPLETE; + } + return OrderedVoteOutcome.RETRY; + } catch (IllegalArgumentException invalidVoteId) { + plugin.debug("Ignored vote receipt release with invalid vote ID"); + return OrderedVoteOutcome.QUARANTINE; + } + } + private void sendSubChannel(GlobalMessageHandler messages, String subChannel, HashMap fields) { JsonEnvelope.Builder builder = JsonEnvelope.builder(subChannel).schema(VotingPluginWire.SCHEMA_VERSION); for (Map.Entry entry : fields.entrySet()) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java index 297515274..7b56a6b05 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java @@ -23,9 +23,10 @@ final class ReliableVoteDeliveryOutbox { private static final long MAX_FILE_BYTES = 16L * 1024L * 1024L; private static final String HEADER = "VP-VOTE-OUTBOX-2"; private static final String ADD = "A"; + private static final String COMPLETED = "C"; private static final String REMOVE = "R"; - record Entry(String server, JsonEnvelope envelope) { } + record Entry(String server, JsonEnvelope envelope, boolean awaitingReceiptRelease) { } private final Path file; private final LinkedHashMap entries = new LinkedHashMap<>(); @@ -43,15 +44,41 @@ synchronized boolean offer(String server, JsonEnvelope envelope) { if (entries.size() >= MAX_ENTRIES) return false; String record = addRecord(server, envelope); if (!prepareAppend(record) || !append(record)) return false; - entries.put(key, new Entry(server, envelope)); + entries.put(key, new Entry(server, envelope, false)); journalRecords++; return true; } - synchronized boolean acknowledge(String server, UUID voteId, String subChannel) { + synchronized boolean acknowledgeCompletion(String server, UUID voteId, String subChannel) { if (voteId == null || server == null || subChannel == null) return false; String key = normalized(server) + '|' + subChannel + '|' + voteId; - if (!entries.containsKey(key)) return false; + Entry entry = entries.get(key); + if (entry == null) return false; + if (entry.awaitingReceiptRelease()) return true; + String record = COMPLETED + '\t' + encode(key) + '\n'; + if (!prepareAppend(record) || !append(record)) return false; + entries.put(key, new Entry(entry.server(), entry.envelope(), true)); + journalRecords++; + return true; + } + + synchronized boolean acknowledgeReceiptRelease(String server, UUID voteId, String subChannel) { + if (voteId == null || server == null || subChannel == null) return false; + String key = normalized(server) + '|' + subChannel + '|' + voteId; + Entry entry = entries.get(key); + if (entry == null || !entry.awaitingReceiptRelease()) return false; + return remove(key); + } + + synchronized boolean acknowledgeLegacyDelivery(String server, UUID voteId, String subChannel) { + if (voteId == null || server == null || subChannel == null) return false; + String key = normalized(server) + '|' + subChannel + '|' + voteId; + Entry entry = entries.get(key); + if (entry == null || entry.awaitingReceiptRelease()) return false; + return remove(key); + } + + private boolean remove(String key) { if (entries.size() == 1) { try { if (!DurableFiles.deleteIfExists(file) && Files.exists(file)) return false; @@ -73,6 +100,14 @@ synchronized List snapshot() { return new ArrayList<>(entries.values()); } + synchronized List pendingVotes() { + return entries.values().stream().filter(entry -> !entry.awaitingReceiptRelease()).toList(); + } + + synchronized List pendingReceiptReleases() { + return entries.values().stream().filter(Entry::awaitingReceiptRelease).toList(); + } + synchronized int size() { return entries.size(); } @@ -86,7 +121,9 @@ private void load() throws IOException { String content = Files.readString(file, StandardCharsets.UTF_8); String[] lines = content.split("\\n", -1); if (lines.length == 0 || !HEADER.equals(lines[0])) throw new IOException("Unsupported vote delivery outbox"); - for (int index = 1; index < lines.length; index++) { + boolean unterminatedTail = !content.endsWith("\n"); + int completeLineLimit = unterminatedTail ? lines.length - 1 : lines.length; + for (int index = 1; index < completeLineLimit; index++) { if (lines[index].isBlank()) continue; String[] parts = lines[index].split("\\t", 3); try { @@ -95,23 +132,22 @@ private void load() throws IOException { JsonEnvelope envelope = JsonEnvelopeCodec.decode(decode(parts[2])); String key = key(server, envelope); if (key == null) throw new IllegalArgumentException("Invalid vote envelope"); - entries.put(key, new Entry(server, envelope)); + entries.put(key, new Entry(server, envelope, false)); + } else if (parts.length == 2 && COMPLETED.equals(parts[0])) { + String key = decode(parts[1]); + Entry entry = entries.get(key); + if (entry == null) throw new IllegalArgumentException("Completion without vote"); + entries.put(key, new Entry(entry.server(), entry.envelope(), true)); } else if (parts.length == 2 && REMOVE.equals(parts[0])) { entries.remove(decode(parts[1])); } else throw new IllegalArgumentException("Unknown journal record"); } catch (RuntimeException malformed) { - // An append interrupted before force may leave only the final record - // truncated. Its caller never observed durable acceptance, so replaying - // the preceding journal is safe and keeps startup available. - if (index == lines.length - 1 && !content.endsWith("\n")) { - if (!compact()) throw new IOException("Unable to repair vote delivery outbox", malformed); - return; - } throw new IOException("Malformed vote delivery outbox entry", malformed); } journalRecords++; if (entries.size() > MAX_ENTRIES) throw new IOException("Vote delivery outbox exceeds entry limit"); } + if (unterminatedTail && !compact()) throw new IOException("Unable to repair vote delivery outbox"); } private boolean prepareAppend(String record) { @@ -150,12 +186,18 @@ private boolean append(String record) { private boolean compact() { try { StringBuilder text = new StringBuilder(HEADER).append('\n'); - for (Entry entry : entries.values()) text.append(addRecord(entry.server(), entry.envelope())); + for (Entry entry : entries.values()) { + text.append(addRecord(entry.server(), entry.envelope())); + if (entry.awaitingReceiptRelease()) { + text.append(COMPLETED).append('\t').append(encode(key(entry.server(), entry.envelope()))).append('\n'); + } + } Path staged = file.resolveSibling(file.getFileName() + ".tmp"); Files.writeString(staged, text, StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); DurableFiles.publishStagedFile(staged, file); - journalRecords = entries.size(); + journalRecords = entries.size() + (int) entries.values().stream() + .filter(Entry::awaitingReceiptRelease).count(); return true; } catch (IOException failure) { return false; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index f5c034d66..a6700ecff 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -911,6 +911,14 @@ private void retryReliableVoteDeliveries(String onlyServer) { for (ReliableVoteDeliveryOutbox.Entry entry : outbox.snapshot()) { if (onlyServer != null && !entry.server().equalsIgnoreCase(onlyServer)) continue; try { + if (entry.awaitingReceiptRelease()) { + if (supportsReliableVoteDelivery(entry.server())) { + String voteId = entry.envelope().getFields().get(VotingPluginWire.K_VOTE_ID); + handler.sendMessage(entry.server(), delay++, VotingPluginWire.voteDeliveryReceiptRelease( + entry.server(), UUID.fromString(voteId), entry.envelope().getSubChannel())); + } + continue; + } if (supportsReliableVoteDelivery(entry.server())) { handler.sendMessage(entry.server(), delay++, VotingPluginWire.requestVoteDeliveryAcknowledgement(entry.envelope())); @@ -922,7 +930,7 @@ private void retryReliableVoteDeliveries(String onlyServer) { } delay++; String voteId = entry.envelope().getFields().get(VotingPluginWire.K_VOTE_ID); - if (!outbox.acknowledge(entry.server(), UUID.fromString(voteId), + if (!outbox.acknowledgeLegacyDelivery(entry.server(), UUID.fromString(voteId), entry.envelope().getSubChannel())) { debug("Legacy vote delivery was accepted but remains queued until its removal is durable for " + entry.server()); @@ -952,14 +960,39 @@ private void handleVoteDeliveryAcknowledgement(JsonEnvelope message) { try { if (!supportsReliableVoteDelivery(server)) return; ReliableVoteDeliveryOutbox outbox = reliableVoteDeliveryOutbox; - if (outbox != null && !outbox.acknowledge(server, UUID.fromString(voteId), subChannel)) { + UUID parsedVoteId = UUID.fromString(voteId); + if (outbox != null && !outbox.acknowledgeCompletion(server, parsedVoteId, subChannel)) { debug("Ignored unmatched or unpersisted vote delivery acknowledgement from " + server); + } else if (outbox != null && globalMessageProxyHandler != null) { + try { + globalMessageProxyHandler.sendMessage(server, 1, + VotingPluginWire.voteDeliveryReceiptRelease(server, parsedVoteId, subChannel)); + } catch (RuntimeException failure) { + debug("Vote receipt release remains queued after the immediate send failed for " + server); + } } } catch (IllegalArgumentException invalidVoteId) { debug("Ignored vote delivery acknowledgement with invalid vote ID from " + server); } } + private void handleVoteDeliveryReceiptReleaseAcknowledgement(JsonEnvelope message) { + if (!VotingPluginWire.advertisesVoteDeliveryAcknowledgement(message)) return; + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + String voteId = message.getFields().getOrDefault(VotingPluginWire.K_VOTE_ID, ""); + String subChannel = message.getFields().getOrDefault(VotingPluginWire.K_VOTE_DELIVERY_SUBCHANNEL, ""); + try { + if (!supportsReliableVoteDelivery(server)) return; + ReliableVoteDeliveryOutbox outbox = reliableVoteDeliveryOutbox; + if (outbox != null && !outbox.acknowledgeReceiptRelease( + server, UUID.fromString(voteId), subChannel)) { + debug("Ignored unmatched or unpersisted vote receipt release acknowledgement from " + server); + } + } catch (IllegalArgumentException invalidVoteId) { + debug("Ignored vote receipt release acknowledgement with invalid vote ID from " + server); + } + } + public synchronized void checkCachedVotes(String server) { int delay = 1; if (isServerValid(server)) { @@ -2075,6 +2108,14 @@ public void onReceive(JsonEnvelope message) { } }); + globalMessageProxyHandler.addListener( + new GlobalMessageListener(VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE_ACK) { + @Override + public void onReceive(JsonEnvelope message) { + handleVoteDeliveryReceiptReleaseAcknowledgement(message); + } + }); + globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { @Override public void onReceive(JsonEnvelope message) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java index aa7c9cefa..911cfeee6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java @@ -44,6 +44,8 @@ private VotingPluginWire() { public static final String SUB_VOTE_DELAY_REJECTED = "VoteDelayRejected"; public static final String SUB_VOTE_BROADCAST = "VoteBroadcast"; public static final String SUB_VOTE_DELIVERY_ACK = "VoteDeliveryAck"; + public static final String SUB_VOTE_DELIVERY_RECEIPT_RELEASE = "VoteDeliveryReceiptRelease"; + public static final String SUB_VOTE_DELIVERY_RECEIPT_RELEASE_ACK = "VoteDeliveryReceiptReleaseAck"; public static final String SUB_BUNGEE_TIME_CHANGE = "BungeeTimeChange"; public static final String SUB_STATUS = "Status"; @@ -111,7 +113,7 @@ private VotingPluginWire() { public static final String K_NUMBER_OF_VOTES = "numberOfVotes"; public static final String K_VOTE_DELIVERY_ACK_VERSION = "voteDeliveryAckVersion"; public static final String K_VOTE_DELIVERY_SUBCHANNEL = "voteDeliverySubchannel"; - public static final int VOTE_DELIVERY_ACK_VERSION = 1; + public static final int VOTE_DELIVERY_ACK_VERSION = 2; /** Origin and receiving proxy names for reliable multi-proxy delivery. */ public static final String K_MULTI_PROXY_ORIGIN = "multiProxyOrigin"; public static final String K_MULTI_PROXY_RECIPIENT = "multiProxyRecipient"; @@ -230,6 +232,23 @@ public static JsonEnvelope voteDeliveryAcknowledgement(String server, UUID voteI .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); } + /** Confirms durable proxy outbox completion so the backend can retire its receipt. */ + public static JsonEnvelope voteDeliveryReceiptRelease(String server, UUID voteId, String voteSubchannel) { + return base(SUB_VOTE_DELIVERY_RECEIPT_RELEASE).put(K_SERVER, safe(server)) + .put(K_VOTE_ID, voteId == null ? "" : voteId.toString()) + .put(K_VOTE_DELIVERY_SUBCHANNEL, safe(voteSubchannel)) + .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); + } + + /** Confirms durable backend receipt retirement to the originating proxy. */ + public static JsonEnvelope voteDeliveryReceiptReleaseAcknowledgement(String server, UUID voteId, + String voteSubchannel) { + return base(SUB_VOTE_DELIVERY_RECEIPT_RELEASE_ACK).put(K_SERVER, safe(server)) + .put(K_VOTE_ID, voteId == null ? "" : voteId.toString()) + .put(K_VOTE_DELIVERY_SUBCHANNEL, safe(voteSubchannel)) + .put(K_VOTE_DELIVERY_ACK_VERSION, VOTE_DELIVERY_ACK_VERSION).build(); + } + public static JsonEnvelope status(String server) { return base(SUB_STATUS).put(K_SERVER, safe(server)).build(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java index a3b87d77b..49582f0ee 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -77,4 +77,17 @@ void receiptFailureFencesCompletedVotePastReservationExpiry() throws Exception { Thread.sleep(5L); assertFalse(cache.reserve(voteId)); } + + @Test + void proxyConfirmedReleaseRetiresReceiptAcrossRestart() { + Path receipts = directory.resolve("receipts.dat"); + UUID voteId = UUID.randomUUID(); + ProcessedVoteCache cache = new ProcessedVoteCache(receipts); + assertTrue(cache.reserve(voteId)); + assertTrue(cache.complete(voteId)); + + assertTrue(cache.releaseCompletedReceipt(voteId)); + + assertTrue(new ProcessedVoteCache(receipts).reserve(voteId)); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java index bc1ce47aa..c39b292ba 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java @@ -26,14 +26,17 @@ import com.bencodez.advancedcore.api.user.AdvancedCoreUser; import com.bencodez.advancedcore.api.user.usercache.UserDataManager; +import com.bencodez.advancedcore.AdvancedCoreConfigOptions; import com.bencodez.simpleapi.scheduler.BukkitScheduler; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import com.bencodez.votingplugin.VotingPluginMain; import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; import com.bencodez.votingplugin.backendproxy.messaging.BackendProxyMessageRouter.OrderedVoteOutcome; import com.bencodez.votingplugin.backendproxy.global.BackendGlobalDataSync; import com.bencodez.votingplugin.backendproxy.presence.BackendPresenceManager; import com.bencodez.votingplugin.backendproxy.voteparty.BackendVotePartySync; +import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.proxy.VotingPluginWire; import com.bencodez.votingplugin.user.UserManager; import com.bencodez.votingplugin.user.VotingPluginUser; @@ -255,4 +258,31 @@ void malformedReliableVoteIsQuarantinedWithoutCompletionReceipt() { anyBoolean(), anyInt()); } + @Test + void receiptReleaseRunsInOrderedLaneAndAcknowledgesDurableRemoval() { + UUID voteId = UUID.randomUUID(); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.releaseCompletedReceipt(voteId)).thenReturn(true); + AdvancedCoreConfigOptions options = mock(AdvancedCoreConfigOptions.class); + when(options.getServer()).thenReturn("survival"); + when(plugin.getOptions()).thenReturn(options); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + BackendProxyMessageRouter voteRouter = new BackendProxyMessageRouter(plugin, + mock(BackendPresenceManager.class), mock(BackendGlobalDataSync.class), + mock(BackendVotePartySync.class), cache); + voteRouter.register(messages, BungeeMethod.REDIS); + AtomicReference outcome = new AtomicReference<>(); + + voteRouter.handleOrderedVote(VotingPluginWire.voteDeliveryReceiptRelease( + "survival", voteId, VotingPluginWire.SUB_VOTE), outcome::set); + + assertEquals(OrderedVoteOutcome.COMPLETE, outcome.get()); + verify(cache).releaseCompletedReceipt(voteId); + org.mockito.ArgumentCaptor acknowledgement = org.mockito.ArgumentCaptor + .forClass(JsonEnvelope.class); + verify(messages).sendMessage(acknowledgement.capture()); + assertEquals(VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE_ACK, + acknowledgement.getValue().getSubChannel()); + } + } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java index 58bd73acd..3d5d05273 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutboxTest.java @@ -32,9 +32,12 @@ void persistsUntilMatchingBackendAcknowledgesCompletion() throws Exception { ReliableVoteDeliveryOutbox restarted = new ReliableVoteDeliveryOutbox(file); assertEquals(1, restarted.size()); - assertFalse(restarted.acknowledge("creative", voteId, VotingPluginWire.SUB_VOTE)); - assertFalse(restarted.acknowledge("survival", voteId, VotingPluginWire.SUB_VOTE_ONLINE)); - assertTrue(restarted.acknowledge("survival", voteId, VotingPluginWire.SUB_VOTE)); + assertFalse(restarted.acknowledgeCompletion("creative", voteId, VotingPluginWire.SUB_VOTE)); + assertFalse(restarted.acknowledgeCompletion("survival", voteId, VotingPluginWire.SUB_VOTE_ONLINE)); + assertTrue(restarted.acknowledgeCompletion("survival", voteId, VotingPluginWire.SUB_VOTE)); + assertEquals(1, restarted.size()); + assertTrue(new ReliableVoteDeliveryOutbox(file).snapshot().get(0).awaitingReceiptRelease()); + assertTrue(restarted.acknowledgeReceiptRelease("survival", voteId, VotingPluginWire.SUB_VOTE)); assertEquals(0, restarted.size()); assertFalse(Files.exists(file)); } @@ -63,7 +66,8 @@ void removalRecordSurvivesRestartWithoutDroppingOtherVotes() throws Exception { "site", 10L, true, true, "", firstId, false, false, 1, 1))); assertTrue(outbox.offer("survival", VotingPluginWire.vote("Two", UUID.randomUUID().toString(), "site", 11L, true, true, "", secondId, false, false, 1, 1))); - assertTrue(outbox.acknowledge("survival", firstId, VotingPluginWire.SUB_VOTE)); + assertTrue(outbox.acknowledgeCompletion("survival", firstId, VotingPluginWire.SUB_VOTE)); + assertTrue(outbox.acknowledgeReceiptRelease("survival", firstId, VotingPluginWire.SUB_VOTE)); ReliableVoteDeliveryOutbox restarted = new ReliableVoteDeliveryOutbox(file); assertEquals(1, restarted.size()); @@ -77,7 +81,7 @@ void restartIgnoresOnlyATruncatedFinalAppend() throws Exception { ReliableVoteDeliveryOutbox outbox = new ReliableVoteDeliveryOutbox(file); assertTrue(outbox.offer("survival", VotingPluginWire.vote("One", UUID.randomUUID().toString(), "site", 10L, true, true, "", UUID.randomUUID(), false, false, 1, 1))); - Files.writeString(file, "A\ttruncated", StandardOpenOption.APPEND); + Files.writeString(file, "R\tc3Vydml2YWw", StandardOpenOption.APPEND); ReliableVoteDeliveryOutbox repaired = new ReliableVoteDeliveryOutbox(file); assertEquals(1, repaired.size()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index 2818e4395..e2f90a7ca 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; @@ -31,6 +32,42 @@ import com.bencodez.votingplugin.tests.VotingPluginProxyTestImpl; class VotingPluginProxyLifecycleTest { + @Test + void completionAckTransitionsThroughDurableReceiptRelease(@TempDir Path directory) throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + GlobalMessageProxyHandler messages = mock(GlobalMessageProxyHandler.class); + UUID voteId = UUID.randomUUID(); + ReliableVoteDeliveryOutbox outbox = new ReliableVoteDeliveryOutbox(directory.resolve("outbox.dat")); + org.junit.jupiter.api.Assertions.assertTrue(outbox.offer("survival", VotingPluginWire.vote( + "Player", UUID.randomUUID().toString(), "site", 10L, true, true, "", voteId, + false, false, 1, 1))); + setField(proxy, "reliableVoteDeliveryOutbox", outbox); + setField(proxy, "globalMessageProxyHandler", messages); + @SuppressWarnings("unchecked") + Set reliable = (Set) field(proxy, "reliableVoteDeliveryServers"); + reliable.add("survival"); + Method completion = VotingPluginProxy.class.getDeclaredMethod( + "handleVoteDeliveryAcknowledgement", JsonEnvelope.class); + completion.setAccessible(true); + completion.invoke(proxy, VotingPluginWire.voteDeliveryAcknowledgement( + "survival", voteId, VotingPluginWire.SUB_VOTE)); + + assertEquals(1, outbox.size()); + org.junit.jupiter.api.Assertions.assertTrue(outbox.snapshot().get(0).awaitingReceiptRelease()); + ArgumentCaptor release = ArgumentCaptor.forClass(JsonEnvelope.class); + verify(messages).sendMessage(org.mockito.ArgumentMatchers.eq("survival"), + org.mockito.ArgumentMatchers.eq(1), release.capture()); + assertEquals(VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE, + release.getValue().getSubChannel()); + + Method released = VotingPluginProxy.class.getDeclaredMethod( + "handleVoteDeliveryReceiptReleaseAcknowledgement", JsonEnvelope.class); + released.setAccessible(true); + released.invoke(proxy, VotingPluginWire.voteDeliveryReceiptReleaseAcknowledgement( + "survival", voteId, VotingPluginWire.SUB_VOTE)); + assertEquals(0, outbox.size()); + } + @Test void drainsPersistedVoteThroughLegacyPathAfterCapabilityDisappears(@TempDir Path directory) throws Exception { VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); @@ -68,6 +105,18 @@ void drainsPersistedVoteThroughLegacyPathAfterCapabilityDisappears(@TempDir Path assertEquals(0, outbox.size()); } + private static void setField(Object target, String name, Object value) throws Exception { + Field field = VotingPluginProxy.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + + private static Object field(Object target, String name) throws Exception { + Field field = VotingPluginProxy.class.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } + @Test void probesPluginMessagingBackendsForDeliveryCapability() throws Exception { VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java index 196d7c686..7d08d48cb 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginWireTest.java @@ -241,6 +241,14 @@ public void voteDeliveryAcknowledgementIsAdditiveAndCorrelated() { assertEquals(voteId.toString(), acknowledgement.getFields().get(VotingPluginWire.K_VOTE_ID)); assertEquals(VotingPluginWire.SUB_VOTE, acknowledgement.getFields().get(VotingPluginWire.K_VOTE_DELIVERY_SUBCHANNEL)); + + JsonEnvelope release = VotingPluginWire.voteDeliveryReceiptRelease( + "survival", voteId, VotingPluginWire.SUB_VOTE); + JsonEnvelope releaseAck = VotingPluginWire.voteDeliveryReceiptReleaseAcknowledgement( + "survival", voteId, VotingPluginWire.SUB_VOTE); + assertEquals(VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE, release.getSubChannel()); + assertEquals(VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE_ACK, releaseAck.getSubChannel()); + assertTrue(VotingPluginWire.requestsVoteDeliveryAcknowledgement(release)); } @Test diff --git a/docs/proxy-vote-delivery.md b/docs/proxy-vote-delivery.md index 2987e4773..5b2a67720 100644 --- a/docs/proxy-vote-delivery.md +++ b/docs/proxy-vote-delivery.md @@ -16,12 +16,17 @@ durable backend overflow queue. Before acknowledgement, the backend journals the completed vote ID without time expiry so a delayed or lost acknowledgement followed by backend restart does not repeat normal completed processing. The bounded receipt journal fails closed at its capacity instead of evicting an ID -that may still have a proxy outbox entry. The proxy then durably -removes the matching server, vote ID, and subchannel entry from its outbox. +that may still have a proxy outbox entry. The proxy then durably transitions the +matching server, vote ID, and subchannel entry into a receipt-release phase. The +backend durably removes its completed receipt and acknowledges that release; +only then does the proxy remove the outbox entry. Lost release messages and +acknowledgements remain retryable across either process restarting, so completed +receipts can be reclaimed without turning the journal bound into a lifetime cap. If a backend generation stops advertising acknowledgements, the proxy drains already-journaled entries once through the existing legacy send path and removes each entry only after the selected transport reports acceptance. This keeps rolling downgrades from -stranding accepted votes while retaining at-least-once behavior. +stranding accepted votes while retaining at-least-once behavior. Entries already +in receipt-release remain until a capable backend confirms retirement. This is an **at least once delivery guarantee**. Proxy shutdown, restart, a lost send, or a lost acknowledgement leaves the outbox entry available for retry. From 617c889a77d4928cda9e9e4d4a7634460e6ec7df Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 06:21:58 -0600 Subject: [PATCH 07/10] Keep capability probes quiet --- .../com/bencodez/votingplugin/proxy/VotingPluginProxy.java | 3 ++- .../votingplugin/proxy/VotingPluginProxyLifecycleTest.java | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index a6700ecff..79892a7cd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -948,7 +948,8 @@ private void probeReliableVoteDeliveryCapabilities() { if (method != BungeeMethod.PLUGINMESSAGING || globalMessageProxyHandler == null) return; int delay = 1; for (String server : getAllAvailableServers()) { - globalMessageProxyHandler.sendMessage(server, delay++, VotingPluginWire.status(server)); + globalMessageProxyHandler.sendMessage(server, delay++, + VotingPluginWire.status(server, UUID.randomUUID())); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index e2f90a7ca..b616e119d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -135,6 +135,8 @@ void probesPluginMessagingBackendsForDeliveryCapability() throws Exception { verify(messages).sendMessage(org.mockito.ArgumentMatchers.eq("survival"), org.mockito.ArgumentMatchers.eq(1), envelope.capture()); assertEquals(VotingPluginWire.SUB_STATUS, envelope.getValue().getSubChannel()); + org.junit.jupiter.api.Assertions.assertFalse(envelope.getValue().getFields() + .get(VotingPluginWire.K_REQUEST_ID).isBlank()); } @Test From 11fac5048781202794a3b8dbf7fa07ed408b8092 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 06:33:46 -0600 Subject: [PATCH 08/10] Fence late vote retries after receipt release --- .mex/events/decisions.jsonl | 1 + AGENTS.md | 3 +- .../cache/DurableVoteReceiptStore.java | 53 +++++++++++-------- .../cache/ProcessedVoteCache.java | 22 ++++++-- .../proxy/ReliableVoteDeliveryOutbox.java | 8 --- .../votingplugin/proxy/VotingPluginProxy.java | 4 +- .../ProcessedVoteCacheDurabilityTest.java | 11 +++- .../proxy/VotingPluginProxyLifecycleTest.java | 5 +- docs/proxy-vote-delivery.md | 13 +++-- 9 files changed, 76 insertions(+), 44 deletions(-) diff --git a/.mex/events/decisions.jsonl b/.mex/events/decisions.jsonl index a169bb1a3..269531cdd 100644 --- a/.mex/events/decisions.jsonl +++ b/.mex/events/decisions.jsonl @@ -2,3 +2,4 @@ {"timestamp":"2026-09-21T11:07:28.400Z","kind":"decision","message":"Reliable backend votes persist completed vote IDs for seven days before acknowledgement. Receipt persistence failure retries without repeating effects in the active process; restart deduplication covers durable completions, while a crash during non-transactional reward effects remains at-least-once.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java"],"cwd":".","source":"agent","status":"implemented"} {"timestamp":"2026-09-21T11:41:29.374Z","kind":"decision","message":"Supersedes the seven-day receipt decision: completion receipts have no time expiry and fail closed at the bounded journal capacity, HTTP participates in the completion-acknowledged outbox, and receipt-write failures keep an in-memory post-effect fence until persistence succeeds.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java"],"cwd":".","source":"agent","status":"implemented"} {"timestamp":"2026-09-21T12:13:09.371Z","kind":"decision","message":"Supersedes indefinite unreclaimed completion receipts: reliable vote delivery uses a durable two-phase receipt-release handshake. After the proxy durably records backend completion, it retries a release marker until the backend durably removes the receipt and acknowledges retirement; only then does the proxy remove the marker.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java"],"cwd":".","source":"agent","status":"implemented"} +{"timestamp":"2026-09-21T12:30:33.599Z","kind":"decision","message":"Supersedes immediate receipt removal after release: backend receipt release creates a 24-hour durable tombstone to fence deliveries already in flight, then reclaims it. Legacy downgrade delivery preserves the proxy release marker so a later capable backend can retire any receipt whose completion acknowledgement was lost.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java"],"cwd":".","source":"agent","status":"implemented"} diff --git a/AGENTS.md b/AGENTS.md index a4861995f..90196f496 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -179,7 +179,8 @@ credentials, generated JARs, dependency caches, IDE output, or unrelated formatt - Proxy-to-backend guaranteed delivery is capability negotiated and at least once. Journal a reward-bearing envelope before reporting transport acceptance, retain it until the matching backend completion acknowledgement is durable, persist completed IDs before acknowledgement for restart-safe deduplication, retire receipts only through the durable - proxy-confirmed release handshake, and keep legacy send behavior for backends that do not advertise the capability. + proxy-confirmed release handshake, retain a bounded durable tombstone for in-flight retries, and keep legacy send + behavior for backends that do not advertise the capability. - Treat scheduler units explicitly. Verify whether each delay is in ticks, milliseconds, or seconds, especially across Bukkit, Folia, BungeeCord, and Velocity adapters. - Register listeners and lifecycle wakeups before producers can publish work; startup/reload ordering must not strand already-persisted or newly-arriving operations. - Protocol-mode changes must not silently broaden legacy v1/RSA acceptance when token-only operation is configured or intended; cover downgrade behavior with tests. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java index e1e9b295a..89cba3371 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import com.bencodez.votingplugin.util.DurableFiles; @@ -19,6 +20,7 @@ final class DurableVoteReceiptStore { private static final long MAX_FILE_BYTES = 16L * 1024L * 1024L; private static final String HEADER = "VP-VOTE-RECEIPTS-1"; private static final String RELEASE = "R"; + static final long RELEASE_TOMBSTONE_TTL_MILLIS = TimeUnit.HOURS.toMillis(24); private static final ConcurrentHashMap FILE_LOCKS = new ConcurrentHashMap<>(); private final Path file; @@ -35,11 +37,13 @@ final class DurableVoteReceiptStore { } synchronized Map snapshot() { + cleanupReleasedTombstones(System.currentTimeMillis()); return new LinkedHashMap<>(receipts); } synchronized long complete(UUID voteId) { if (voteId == null) return 0L; + cleanupReleasedTombstones(System.currentTimeMillis()); Long current = receipts.get(voteId); if (current != null) return current; if (receipts.size() >= MAX_RECEIPTS) return 0L; @@ -53,27 +57,21 @@ synchronized long complete(UUID voteId) { return expiresAt; } - synchronized boolean release(UUID voteId) { - if (voteId == null || !receipts.containsKey(voteId)) return true; - if (receipts.size() == 1) { - synchronized (fileLock) { - try { - if (!DurableFiles.deleteIfExists(file) && Files.exists(file)) return false; - } catch (IOException failure) { - return false; - } - } - receipts.clear(); - journalRecords = 0; - return true; - } - String record = RELEASE + '\t' + voteId + '\n'; + synchronized long release(UUID voteId) { + if (voteId == null) return 0L; + long now = System.currentTimeMillis(); + cleanupReleasedTombstones(now); + Long current = receipts.get(voteId); + if (current == null) return Long.MAX_VALUE; + if (current != Long.MAX_VALUE) return current; + long expiresAt = now + RELEASE_TOMBSTONE_TTL_MILLIS; + String record = RELEASE + '\t' + voteId + '\t' + expiresAt + '\n'; synchronized (fileLock) { - if (!prepareAppend(record) || !append(record)) return false; + if (!prepareAppend(record) || !append(record)) return 0L; } - receipts.remove(voteId); + receipts.put(voteId, expiresAt); journalRecords++; - return true; + return expiresAt; } private void load() throws IOException { @@ -90,10 +88,17 @@ private void load() throws IOException { for (int index = 1; index < completeLineLimit; index++) { if (lines[index].isBlank()) continue; try { - String[] fields = lines[index].split("\\t", 2); - if (fields.length != 2) throw new IllegalArgumentException("Malformed receipt"); - if (RELEASE.equals(fields[0])) receipts.remove(UUID.fromString(fields[1])); - else { + String[] fields = lines[index].split("\\t", 3); + if (RELEASE.equals(fields[0])) { + if (fields.length == 2) receipts.remove(UUID.fromString(fields[1])); + else if (fields.length == 3) { + UUID voteId = UUID.fromString(fields[1]); + long expiresAt = Long.parseLong(fields[2]); + if (expiresAt > System.currentTimeMillis()) receipts.put(voteId, expiresAt); + else receipts.remove(voteId); + } else throw new IllegalArgumentException("Malformed receipt release"); + } else { + if (fields.length != 2) throw new IllegalArgumentException("Malformed receipt"); UUID voteId = UUID.fromString(fields[0]); long expiresAt = Long.parseLong(fields[1]); receipts.put(voteId, expiresAt); @@ -156,4 +161,8 @@ private boolean compact() { return false; } } + + private void cleanupReleasedTombstones(long now) { + receipts.entrySet().removeIf(entry -> entry.getValue() != Long.MAX_VALUE && entry.getValue() <= now); + } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index 1ba5e948d..dd4bcfedb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -92,6 +92,7 @@ public boolean reserve(UUID voteId) { } if (processedVotes.replace(voteId, currentExpiry, expiresAt)) { + completedVotes.remove(voteId); cleanup(now); return true; } @@ -123,9 +124,20 @@ public boolean hasCompletedEffects(UUID voteId) { /** Durably retires a completed receipt after the proxy confirms outbox removal. */ public boolean releaseCompletedReceipt(UUID voteId) { if (voteId == null) return false; - if (durableReceipts != null && !durableReceipts.release(voteId)) return false; - completedVotes.remove(voteId); - processedVotes.remove(voteId); + if (durableReceipts == null) { + completedVotes.remove(voteId); + processedVotes.remove(voteId); + return true; + } + long expiresAt = durableReceipts.release(voteId); + if (expiresAt <= 0L) return false; + if (expiresAt == Long.MAX_VALUE) { + completedVotes.remove(voteId); + processedVotes.remove(voteId); + } else { + completedVotes.add(voteId); + processedVotes.put(voteId, expiresAt); + } return true; } @@ -230,7 +242,9 @@ public synchronized void unregisterRedisSubscriber(Object subscriber) { } private void cleanup(long now) { - processedVotes.entrySet().removeIf(entry -> entry.getValue() <= now); + processedVotes.forEach((voteId, expiresAt) -> { + if (expiresAt <= now && processedVotes.remove(voteId, expiresAt)) completedVotes.remove(voteId); + }); } /** Returns an exact UTF-8 length up to the per-delivery cap, then cap + 1. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java index 7b56a6b05..13e372b7c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java @@ -70,14 +70,6 @@ synchronized boolean acknowledgeReceiptRelease(String server, UUID voteId, Strin return remove(key); } - synchronized boolean acknowledgeLegacyDelivery(String server, UUID voteId, String subChannel) { - if (voteId == null || server == null || subChannel == null) return false; - String key = normalized(server) + '|' + subChannel + '|' + voteId; - Entry entry = entries.get(key); - if (entry == null || entry.awaitingReceiptRelease()) return false; - return remove(key); - } - private boolean remove(String key) { if (entries.size() == 1) { try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 79892a7cd..0e9a2ad39 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -930,9 +930,9 @@ private void retryReliableVoteDeliveries(String onlyServer) { } delay++; String voteId = entry.envelope().getFields().get(VotingPluginWire.K_VOTE_ID); - if (!outbox.acknowledgeLegacyDelivery(entry.server(), UUID.fromString(voteId), + if (!outbox.acknowledgeCompletion(entry.server(), UUID.fromString(voteId), entry.envelope().getSubChannel())) { - debug("Legacy vote delivery was accepted but remains queued until its removal is durable for " + debug("Legacy vote delivery was accepted but remains queued until its release state is durable for " + entry.server()); } } else { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java index 49582f0ee..f95e74d03 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -79,7 +79,7 @@ void receiptFailureFencesCompletedVotePastReservationExpiry() throws Exception { } @Test - void proxyConfirmedReleaseRetiresReceiptAcrossRestart() { + void proxyConfirmedReleaseLeavesRestartSafeTombstone() { Path receipts = directory.resolve("receipts.dat"); UUID voteId = UUID.randomUUID(); ProcessedVoteCache cache = new ProcessedVoteCache(receipts); @@ -88,6 +88,15 @@ void proxyConfirmedReleaseRetiresReceiptAcrossRestart() { assertTrue(cache.releaseCompletedReceipt(voteId)); + assertFalse(new ProcessedVoteCache(receipts).reserve(voteId)); + } + + @Test + void expiredReleaseTombstoneIsReclaimedOnRestart() throws Exception { + Path receipts = directory.resolve("receipts.dat"); + UUID voteId = UUID.randomUUID(); + Files.writeString(receipts, "VP-VOTE-RECEIPTS-1\nR\t" + voteId + "\t1\n"); + assertTrue(new ProcessedVoteCache(receipts).reserve(voteId)); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index b616e119d..ac86e188d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -69,7 +69,7 @@ void completionAckTransitionsThroughDurableReceiptRelease(@TempDir Path director } @Test - void drainsPersistedVoteThroughLegacyPathAfterCapabilityDisappears(@TempDir Path directory) throws Exception { + void preservesReceiptReleaseAfterLegacyDowngradeDelivery(@TempDir Path directory) throws Exception { VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); proxy.setMethod(BungeeMethod.PLUGINMESSAGING); GlobalMessageProxyHandler messages = mock(GlobalMessageProxyHandler.class); @@ -102,7 +102,8 @@ void drainsPersistedVoteThroughLegacyPathAfterCapabilityDisappears(@TempDir Path proxy.setPluginMessageDeliveryResult(true); retry.invoke(proxy, "survival"); - assertEquals(0, outbox.size()); + assertEquals(1, outbox.size()); + org.junit.jupiter.api.Assertions.assertTrue(outbox.snapshot().get(0).awaitingReceiptRelease()); } private static void setField(Object target, String name, Object value) throws Exception { diff --git a/docs/proxy-vote-delivery.md b/docs/proxy-vote-delivery.md index 5b2a67720..a66fa5b2e 100644 --- a/docs/proxy-vote-delivery.md +++ b/docs/proxy-vote-delivery.md @@ -18,20 +18,25 @@ followed by backend restart does not repeat normal completed processing. The bounded receipt journal fails closed at its capacity instead of evicting an ID that may still have a proxy outbox entry. The proxy then durably transitions the matching server, vote ID, and subchannel entry into a receipt-release phase. The -backend durably removes its completed receipt and acknowledges that release; +backend durably converts its completed receipt into a 24-hour post-release +tombstone and acknowledges that release; only then does the proxy remove the outbox entry. Lost release messages and acknowledgements remain retryable across either process restarting, so completed receipts can be reclaimed without turning the journal bound into a lifetime cap. If a backend generation stops advertising acknowledgements, the proxy drains already-journaled entries once through the existing legacy send path and removes each entry only after the selected transport reports acceptance. This keeps rolling downgrades from -stranding accepted votes while retaining at-least-once behavior. Entries already -in receipt-release remain until a capable backend confirms retirement. +stranding accepted votes while retaining at-least-once behavior. The accepted +entry transitions into receipt release in case the previous capable backend +journaled completion before its acknowledgement was lost. Receipt-release +entries remain until a capable backend confirms retirement. This is an **at least once delivery guarantee**. Proxy shutdown, restart, a lost send, or a lost acknowledgement leaves the outbox entry available for retry. The backend vote ID cache and durable completion journal suppress ordinary and -restart-spanning duplicate retries. A backend process crash during reward side +restart-spanning duplicate retries. The bounded post-release tombstone also +fences transport retries that were already in flight when completion was +acknowledged. A backend process crash during reward side effects, before the completion record is durable, can still lead to a repeated attempt because reward execution and the receipt cannot be committed atomically. Stronger exactly once reward execution would require a separate reward API and From dd1a2a574f250bf40abcff551286ba938e929870 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 06:57:07 -0600 Subject: [PATCH 09/10] Keep receipt release progress bounded --- .mex/events/decisions.jsonl | 1 + .../backendproxy/BackendProxyHandler.java | 37 +++++++++++ .../cache/DurableVoteReceiptStore.java | 66 +++++++++++++++---- .../cache/ProcessedVoteCache.java | 14 ++-- .../messaging/BackendProxyMessageRouter.java | 32 ++++++--- .../BackendProxyHandlerLifecycleTest.java | 36 ++++++++++ .../ProcessedVoteCacheDurabilityTest.java | 30 +++++++++ .../BackendProxyMessageRouterTest.java | 8 ++- docs/proxy-vote-delivery.md | 13 ++-- 9 files changed, 204 insertions(+), 33 deletions(-) diff --git a/.mex/events/decisions.jsonl b/.mex/events/decisions.jsonl index 269531cdd..622406816 100644 --- a/.mex/events/decisions.jsonl +++ b/.mex/events/decisions.jsonl @@ -3,3 +3,4 @@ {"timestamp":"2026-09-21T11:41:29.374Z","kind":"decision","message":"Supersedes the seven-day receipt decision: completion receipts have no time expiry and fail closed at the bounded journal capacity, HTTP participates in the completion-acknowledged outbox, and receipt-write failures keep an in-memory post-effect fence until persistence succeeds.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java"],"cwd":".","source":"agent","status":"implemented"} {"timestamp":"2026-09-21T12:13:09.371Z","kind":"decision","message":"Supersedes indefinite unreclaimed completion receipts: reliable vote delivery uses a durable two-phase receipt-release handshake. After the proxy durably records backend completion, it retries a release marker until the backend durably removes the receipt and acknowledges retirement; only then does the proxy remove the marker.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/ReliableVoteDeliveryOutbox.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java"],"cwd":".","source":"agent","status":"implemented"} {"timestamp":"2026-09-21T12:30:33.599Z","kind":"decision","message":"Supersedes immediate receipt removal after release: backend receipt release creates a 24-hour durable tombstone to fence deliveries already in flight, then reclaims it. Legacy downgrade delivery preserves the proxy release marker so a later capable backend can retire any receipt whose completion acknowledgement was lost.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java"],"cwd":".","source":"agent","status":"implemented"} +{"timestamp":"2026-09-21T12:46:34.284Z","kind":"decision","message":"Receipt releases persist a 24-hour tombstone even when the backend has no active receipt, fencing a late legacy retry. The bounded receipt store keeps completion headroom larger than the entire ordered lane plus separate tombstone capacity. Releases backed by an existing durable receipt may bypass a capacity-blocked vote because durable completion makes that reordering safe; unknown releases remain ordered.","files":["VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java","VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java"],"cwd":".","source":"agent","status":"implemented"} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index ce2efb6aa..af2904813 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -56,6 +56,7 @@ public class BackendProxyHandler implements Listener { private boolean orderedVoteDispatchPaused = true; private boolean orderedVoteDispatchClosing; private boolean orderedVoteQuarantineFailed; + private final AtomicBoolean durableReceiptReleaseActive = new AtomicBoolean(); private JsonEnvelope orderedVoteDispatchInFlight; private BackendOrderedVoteOverflowQueue.PendingEnvelope orderedVoteOverflowInFlight; private JsonEnvelope orderedVoteShutdownQuarantined; @@ -234,6 +235,42 @@ private boolean isOrderedVoteMessage(JsonEnvelope envelope) { } private void dispatchOrderedVote(JsonEnvelope envelope, Runnable ignoredLocalDispatch) { + BackendProxyMessageRouter router = messageRouter; + if (router != null && router.hasDurableReceiptForRelease(envelope) + && durableReceiptReleaseActive.compareAndSet(false, true)) { + try { + plugin.getBukkitScheduler().runTaskAsynchronously(plugin, + () -> processDurableReceiptRelease(router, envelope, ignoredLocalDispatch)); + return; + } catch (RuntimeException schedulingFailure) { + plugin.debug(schedulingFailure); + processDurableReceiptRelease(router, envelope, ignoredLocalDispatch); + return; + } + } + enqueueOrderedVote(envelope, ignoredLocalDispatch); + } + + private void processDurableReceiptRelease(BackendProxyMessageRouter router, JsonEnvelope envelope, + Runnable ignoredLocalDispatch) { + AtomicBoolean completed = new AtomicBoolean(); + java.util.function.Consumer completion = outcome -> { + if (!completed.compareAndSet(false, true)) return; + durableReceiptReleaseActive.set(false); + if (outcome != OrderedVoteOutcome.COMPLETE) enqueueOrderedVote(envelope, ignoredLocalDispatch); + }; + try { + router.handleOrderedVote(envelope, completion); + } catch (RuntimeException | Error failure) { + if (completed.compareAndSet(false, true)) { + durableReceiptReleaseActive.set(false); + enqueueOrderedVote(envelope, ignoredLocalDispatch); + } + throw failure; + } + } + + private void enqueueOrderedVote(JsonEnvelope envelope, Runnable ignoredLocalDispatch) { BackendProxyHandler handoffTarget; synchronized (orderedVoteDispatch) { handoffTarget = orderedVoteHandoffTarget; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java index 89cba3371..136fe9a53 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -16,7 +16,11 @@ /** Bounded append journal for backend vote IDs completed before acknowledgement. */ final class DurableVoteReceiptStore { - private static final int MAX_RECEIPTS = 262144; + private static final int MAX_ACTIVE_RECEIPTS = 262144; + /* Larger than the complete in-memory and durable ordered lane (256 + 512). */ + private static final int COMPLETION_HEADROOM = 1024; + /* One proxy cannot retain more release markers than its bounded outbox. */ + private static final int MAX_RELEASE_TOMBSTONES = 4096; private static final long MAX_FILE_BYTES = 16L * 1024L * 1024L; private static final String HEADER = "VP-VOTE-RECEIPTS-1"; private static final String RELEASE = "R"; @@ -25,12 +29,25 @@ final class DurableVoteReceiptStore { private final Path file; private final Object fileLock; + private final int maxActiveReceipts; + private final int completionHeadroom; + private final int maxReleaseTombstones; private final LinkedHashMap receipts = new LinkedHashMap<>(); + private int activeReceipts; + private int releaseTombstones; private int journalRecords; DurableVoteReceiptStore(Path file) throws IOException { + this(file, MAX_ACTIVE_RECEIPTS, COMPLETION_HEADROOM, MAX_RELEASE_TOMBSTONES); + } + + DurableVoteReceiptStore(Path file, int maxActiveReceipts, int completionHeadroom, + int maxReleaseTombstones) throws IOException { this.file = file.toAbsolutePath().normalize(); this.fileLock = FILE_LOCKS.computeIfAbsent(this.file, ignored -> new Object()); + this.maxActiveReceipts = maxActiveReceipts; + this.completionHeadroom = completionHeadroom; + this.maxReleaseTombstones = maxReleaseTombstones; synchronized (fileLock) { load(); } @@ -41,18 +58,23 @@ synchronized Map snapshot() { return new LinkedHashMap<>(receipts); } + synchronized boolean contains(UUID voteId) { + cleanupReleasedTombstones(System.currentTimeMillis()); + return voteId != null && receipts.containsKey(voteId); + } + synchronized long complete(UUID voteId) { if (voteId == null) return 0L; cleanupReleasedTombstones(System.currentTimeMillis()); Long current = receipts.get(voteId); if (current != null) return current; - if (receipts.size() >= MAX_RECEIPTS) return 0L; + if (activeReceipts >= maxActiveReceipts + completionHeadroom) return 0L; long expiresAt = Long.MAX_VALUE; String record = voteId + "\t" + expiresAt + '\n'; synchronized (fileLock) { if (!prepareAppend(record) || !append(record)) return 0L; } - receipts.put(voteId, expiresAt); + putReceipt(voteId, expiresAt); journalRecords++; return expiresAt; } @@ -62,14 +84,14 @@ synchronized long release(UUID voteId) { long now = System.currentTimeMillis(); cleanupReleasedTombstones(now); Long current = receipts.get(voteId); - if (current == null) return Long.MAX_VALUE; - if (current != Long.MAX_VALUE) return current; + if (current == null && releaseTombstones >= maxReleaseTombstones) return 0L; + if (current != null && current != Long.MAX_VALUE) return current; long expiresAt = now + RELEASE_TOMBSTONE_TTL_MILLIS; String record = RELEASE + '\t' + voteId + '\t' + expiresAt + '\n'; synchronized (fileLock) { if (!prepareAppend(record) || !append(record)) return 0L; } - receipts.put(voteId, expiresAt); + putReceipt(voteId, expiresAt); journalRecords++; return expiresAt; } @@ -90,24 +112,27 @@ private void load() throws IOException { try { String[] fields = lines[index].split("\\t", 3); if (RELEASE.equals(fields[0])) { - if (fields.length == 2) receipts.remove(UUID.fromString(fields[1])); + if (fields.length == 2) removeReceipt(UUID.fromString(fields[1])); else if (fields.length == 3) { UUID voteId = UUID.fromString(fields[1]); long expiresAt = Long.parseLong(fields[2]); - if (expiresAt > System.currentTimeMillis()) receipts.put(voteId, expiresAt); - else receipts.remove(voteId); + if (expiresAt > System.currentTimeMillis()) putReceipt(voteId, expiresAt); + else removeReceipt(voteId); } else throw new IllegalArgumentException("Malformed receipt release"); } else { if (fields.length != 2) throw new IllegalArgumentException("Malformed receipt"); UUID voteId = UUID.fromString(fields[0]); long expiresAt = Long.parseLong(fields[1]); - receipts.put(voteId, expiresAt); + putReceipt(voteId, expiresAt); } } catch (RuntimeException malformed) { throw new IOException("Malformed vote receipt journal", malformed); } journalRecords++; - if (receipts.size() > MAX_RECEIPTS) throw new IOException("Vote receipt journal exceeds entry limit"); + if (activeReceipts > maxActiveReceipts + completionHeadroom + || releaseTombstones > maxReleaseTombstones) { + throw new IOException("Vote receipt journal exceeds entry limit"); + } } if (unterminatedTail && !compact()) throw new IOException("Unable to repair vote receipt journal"); } @@ -163,6 +188,25 @@ private boolean compact() { } private void cleanupReleasedTombstones(long now) { + int before = receipts.size(); receipts.entrySet().removeIf(entry -> entry.getValue() != Long.MAX_VALUE && entry.getValue() <= now); + releaseTombstones -= before - receipts.size(); + } + + private void putReceipt(UUID voteId, long expiresAt) { + Long previous = receipts.put(voteId, expiresAt); + if (previous != null) { + if (previous == Long.MAX_VALUE) activeReceipts--; + else releaseTombstones--; + } + if (expiresAt == Long.MAX_VALUE) activeReceipts++; + else releaseTombstones++; + } + + private void removeReceipt(UUID voteId) { + Long previous = receipts.remove(voteId); + if (previous == null) return; + if (previous == Long.MAX_VALUE) activeReceipts--; + else releaseTombstones--; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index dd4bcfedb..ccc8f8cdf 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -121,6 +121,11 @@ public boolean hasCompletedEffects(UUID voteId) { return voteId != null && (completedVotes.contains(voteId) || completedAwaitingReceipt.contains(voteId)); } + /** Returns whether an acknowledgement-safe receipt is already durable. */ + public boolean hasDurableReceipt(UUID voteId) { + return durableReceipts != null && durableReceipts.contains(voteId); + } + /** Durably retires a completed receipt after the proxy confirms outbox removal. */ public boolean releaseCompletedReceipt(UUID voteId) { if (voteId == null) return false; @@ -131,13 +136,8 @@ public boolean releaseCompletedReceipt(UUID voteId) { } long expiresAt = durableReceipts.release(voteId); if (expiresAt <= 0L) return false; - if (expiresAt == Long.MAX_VALUE) { - completedVotes.remove(voteId); - processedVotes.remove(voteId); - } else { - completedVotes.add(voteId); - processedVotes.put(voteId, expiresAt); - } + completedVotes.add(voteId); + processedVotes.put(voteId, expiresAt); return true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java index 84e750b0a..93dbd3594 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouter.java @@ -159,6 +159,12 @@ public void handleOrderedVote(JsonEnvelope msg, Consumer com throw new IllegalArgumentException("Unsupported ordered proxy vote message: " + subChannel); } + /** Allows a durable receipt release to bypass a capacity-blocked ordered vote. */ + public boolean hasDurableReceiptForRelease(JsonEnvelope msg) { + UUID voteId = validReceiptReleaseVoteId(msg); + return voteId != null && processedVoteCache.hasDurableReceipt(voteId); + } + /** * Processes one ordered VoteUpdate. User identity and shared cache population * are allowed to leave the platform thread, while offline reward/Bukkit work @@ -428,16 +434,11 @@ private boolean validSchema(JsonEnvelope msg) { } private OrderedVoteOutcome handleVoteDeliveryReceiptRelease(GlobalMessageHandler messages, JsonEnvelope msg) { - if (messages == null || !VotingPluginWire.requestsVoteDeliveryAcknowledgement(msg)) { - return OrderedVoteOutcome.QUARANTINE; - } - String server = nvl(msg.getFields().get(VotingPluginWire.K_SERVER)); - if (!plugin.getOptions().getServer().equalsIgnoreCase(server)) return OrderedVoteOutcome.QUARANTINE; + if (messages == null) return OrderedVoteOutcome.QUARANTINE; + UUID voteId = validReceiptReleaseVoteId(msg); + if (voteId == null) return OrderedVoteOutcome.QUARANTINE; String subChannel = nvl(msg.getFields().get(VotingPluginWire.K_VOTE_DELIVERY_SUBCHANNEL)); - if (!VotingPluginWire.SUB_VOTE.equals(subChannel) - && !VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel)) return OrderedVoteOutcome.QUARANTINE; try { - UUID voteId = UUID.fromString(nvl(msg.getFields().get(VotingPluginWire.K_VOTE_ID))); if (processedVoteCache.releaseCompletedReceipt(voteId)) { messages.sendMessage(VotingPluginWire.voteDeliveryReceiptReleaseAcknowledgement( plugin.getOptions().getServer(), voteId, subChannel)); @@ -450,6 +451,21 @@ private OrderedVoteOutcome handleVoteDeliveryReceiptRelease(GlobalMessageHandler } } + private UUID validReceiptReleaseVoteId(JsonEnvelope msg) { + if (msg == null || !VotingPluginWire.SUB_VOTE_DELIVERY_RECEIPT_RELEASE.equals(msg.getSubChannel()) + || !VotingPluginWire.requestsVoteDeliveryAcknowledgement(msg)) return null; + String server = nvl(msg.getFields().get(VotingPluginWire.K_SERVER)); + if (!plugin.getOptions().getServer().equalsIgnoreCase(server)) return null; + String subChannel = nvl(msg.getFields().get(VotingPluginWire.K_VOTE_DELIVERY_SUBCHANNEL)); + if (!VotingPluginWire.SUB_VOTE.equals(subChannel) + && !VotingPluginWire.SUB_VOTE_ONLINE.equals(subChannel)) return null; + try { + return UUID.fromString(nvl(msg.getFields().get(VotingPluginWire.K_VOTE_ID))); + } catch (IllegalArgumentException invalidVoteId) { + return null; + } + } + private void sendSubChannel(GlobalMessageHandler messages, String subChannel, HashMap fields) { JsonEnvelope.Builder builder = JsonEnvelope.builder(subChannel).schema(VotingPluginWire.SCHEMA_VERSION); for (Map.Entry entry : fields.entrySet()) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 3145fa538..80a82ad99 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -145,6 +145,42 @@ void voteAndVoteUpdateMessagesStayOrderedAcrossAsyncAndPlatformWork() throws Exc verify(scheduler, times(3)).runTaskAsynchronously(eq(plugin), any(Runnable.class)); } + @Test + void durableReceiptReleaseBypassesBlockedOrderedVote() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + BackendProxyHandler handler = new BackendProxyHandler(plugin); + BackendProxyMessageRouter router = mock(BackendProxyMessageRouter.class); + setField(handler, "messageRouter", router); + handler.activateInboundMessages(); + + ArrayDeque asyncTasks = new ArrayDeque<>(); + doAnswer(invocation -> { + asyncTasks.addLast(invocation.getArgument(1)); + return null; + }).when(scheduler).runTaskAsynchronously(eq(plugin), any(Runnable.class)); + + JsonEnvelope blockedVote = JsonEnvelope.builder(VotingPluginWire.SUB_VOTE).build(); + JsonEnvelope release = VotingPluginWire.voteDeliveryReceiptRelease( + "survival", UUID.randomUUID(), VotingPluginWire.SUB_VOTE); + JsonEnvelope secondRelease = VotingPluginWire.voteDeliveryReceiptRelease( + "survival", UUID.randomUUID(), VotingPluginWire.SUB_VOTE); + when(router.hasDurableReceiptForRelease(release)).thenReturn(true); + when(router.hasDurableReceiptForRelease(secondRelease)).thenReturn(true); + + handler.dispatchIncomingAfterPublication(blockedVote, mock(Runnable.class)); + handler.dispatchIncomingAfterPublication(release, mock(Runnable.class)); + handler.dispatchIncomingAfterPublication(secondRelease, mock(Runnable.class)); + + assertEquals(2, asyncTasks.size(), "only one known release should bypass the active ordered head"); + asyncTasks.removeLast().run(); + verify(router).handleOrderedVote(eq(release), any()); + @SuppressWarnings("unchecked") + ArrayDeque queued = (ArrayDeque) getField(handler, "orderedVoteDispatchQueue"); + assertEquals(java.util.List.of(blockedVote, secondRelease), java.util.List.copyOf(queued)); + } + @Test void reliableVoteIsAcknowledgedOnlyAfterOrderedCompletion() throws Exception { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java index f95e74d03..35aa03f72 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -23,6 +23,7 @@ void completedVoteRemainsDeduplicatedAfterBackendRestart() { assertTrue(first.reserve(voteId)); assertTrue(first.complete(voteId)); + assertTrue(first.hasDurableReceipt(voteId)); assertFalse(new ProcessedVoteCache(receipts).reserve(voteId)); } @@ -91,6 +92,35 @@ void proxyConfirmedReleaseLeavesRestartSafeTombstone() { assertFalse(new ProcessedVoteCache(receipts).reserve(voteId)); } + @Test + void unknownReceiptReleaseLeavesRestartSafeTombstone() { + Path receipts = directory.resolve("receipts.dat"); + UUID voteId = UUID.randomUUID(); + ProcessedVoteCache cache = new ProcessedVoteCache(receipts); + + assertFalse(cache.hasDurableReceipt(voteId)); + assertTrue(cache.releaseCompletedReceipt(voteId)); + assertTrue(cache.hasDurableReceipt(voteId)); + + assertFalse(new ProcessedVoteCache(receipts).reserve(voteId)); + } + + @Test + void completionHeadroomAllowsPrecedingVotesBeforeRelease() throws Exception { + DurableVoteReceiptStore store = new DurableVoteReceiptStore(directory.resolve("receipts.dat"), 1, 2, 1); + UUID first = UUID.randomUUID(); + UUID second = UUID.randomUUID(); + UUID third = UUID.randomUUID(); + UUID fourth = UUID.randomUUID(); + + assertTrue(store.complete(first) > 0L); + assertTrue(store.complete(second) > 0L); + assertTrue(store.complete(third) > 0L); + assertFalse(store.complete(fourth) > 0L); + assertTrue(store.release(first) > 0L); + assertTrue(store.complete(fourth) > 0L); + } + @Test void expiredReleaseTombstoneIsReclaimedOnRestart() throws Exception { Path receipts = directory.resolve("receipts.dat"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java index c39b292ba..266f65afa 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/messaging/BackendProxyMessageRouterTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -262,6 +263,7 @@ void malformedReliableVoteIsQuarantinedWithoutCompletionReceipt() { void receiptReleaseRunsInOrderedLaneAndAcknowledgesDurableRemoval() { UUID voteId = UUID.randomUUID(); ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.hasDurableReceipt(voteId)).thenReturn(true); when(cache.releaseCompletedReceipt(voteId)).thenReturn(true); AdvancedCoreConfigOptions options = mock(AdvancedCoreConfigOptions.class); when(options.getServer()).thenReturn("survival"); @@ -273,8 +275,10 @@ void receiptReleaseRunsInOrderedLaneAndAcknowledgesDurableRemoval() { voteRouter.register(messages, BungeeMethod.REDIS); AtomicReference outcome = new AtomicReference<>(); - voteRouter.handleOrderedVote(VotingPluginWire.voteDeliveryReceiptRelease( - "survival", voteId, VotingPluginWire.SUB_VOTE), outcome::set); + JsonEnvelope release = VotingPluginWire.voteDeliveryReceiptRelease( + "survival", voteId, VotingPluginWire.SUB_VOTE); + assertTrue(voteRouter.hasDurableReceiptForRelease(release)); + voteRouter.handleOrderedVote(release, outcome::set); assertEquals(OrderedVoteOutcome.COMPLETE, outcome.get()); verify(cache).releaseCompletedReceipt(voteId); diff --git a/docs/proxy-vote-delivery.md b/docs/proxy-vote-delivery.md index a66fa5b2e..05bb50d42 100644 --- a/docs/proxy-vote-delivery.md +++ b/docs/proxy-vote-delivery.md @@ -16,7 +16,11 @@ durable backend overflow queue. Before acknowledgement, the backend journals the completed vote ID without time expiry so a delayed or lost acknowledgement followed by backend restart does not repeat normal completed processing. The bounded receipt journal fails closed at its capacity instead of evicting an ID -that may still have a proxy outbox entry. The proxy then durably transitions the +that may still have a proxy outbox entry. It reserves completion headroom larger +than the bounded ordered lane and separate space for release tombstones. A +release for an already-durable receipt may run independently of a +capacity-blocked vote because that receipt proves the vote effects completed; +unknown releases remain ordered. The proxy then durably transitions the matching server, vote ID, and subchannel entry into a receipt-release phase. The backend durably converts its completed receipt into a 24-hour post-release tombstone and acknowledges that release; @@ -24,10 +28,9 @@ only then does the proxy remove the outbox entry. Lost release messages and acknowledgements remain retryable across either process restarting, so completed receipts can be reclaimed without turning the journal bound into a lifetime cap. If a backend generation stops advertising acknowledgements, the proxy drains -already-journaled entries once through the existing legacy send path and removes -each entry only after the selected transport reports acceptance. This keeps rolling downgrades from -stranding accepted votes while retaining at-least-once behavior. The accepted -entry transitions into receipt release in case the previous capable backend +already-journaled entries once through the existing legacy send path and moves +each accepted entry into receipt release. This keeps rolling downgrades from +stranding accepted votes while retaining at-least-once behavior in case the previous capable backend journaled completion before its acknowledgement was lost. Receipt-release entries remain until a capable backend confirms retirement. From e3467f3a223edcd7137ba1e2408fd06d91468347 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:12:55 -0600 Subject: [PATCH 10/10] Enforce receipt tombstone capacity --- .../backendproxy/cache/DurableVoteReceiptStore.java | 2 +- .../cache/ProcessedVoteCacheDurabilityTest.java | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java index 136fe9a53..50551a6a2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/DurableVoteReceiptStore.java @@ -84,8 +84,8 @@ synchronized long release(UUID voteId) { long now = System.currentTimeMillis(); cleanupReleasedTombstones(now); Long current = receipts.get(voteId); - if (current == null && releaseTombstones >= maxReleaseTombstones) return 0L; if (current != null && current != Long.MAX_VALUE) return current; + if (releaseTombstones >= maxReleaseTombstones) return 0L; long expiresAt = now + RELEASE_TOMBSTONE_TTL_MILLIS; String record = RELEASE + '\t' + voteId + '\t' + expiresAt + '\n'; synchronized (fileLock) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java index 35aa03f72..0bc4d8b8c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCacheDurabilityTest.java @@ -121,6 +121,19 @@ void completionHeadroomAllowsPrecedingVotesBeforeRelease() throws Exception { assertTrue(store.complete(fourth) > 0L); } + @Test + void activeReceiptWaitsWhenTombstoneCapacityIsFull() throws Exception { + Path receipts = directory.resolve("receipts.dat"); + DurableVoteReceiptStore store = new DurableVoteReceiptStore(receipts, 1, 0, 1); + UUID active = UUID.randomUUID(); + assertTrue(store.complete(active) > 0L); + assertTrue(store.release(UUID.randomUUID()) > 0L); + + assertFalse(store.release(active) > 0L); + + new DurableVoteReceiptStore(receipts, 1, 0, 1); + } + @Test void expiredReleaseTombstoneIsReclaimedOnRestart() throws Exception { Path receipts = directory.resolve("receipts.dat");