From 88f72c64228393ebeb128138e875158b4baa194d Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 2 Sep 2026 14:23:37 +0800 Subject: [PATCH 1/8] [Feature] Add audit hooks for inter-node user data transfers --- .../consensus/config/ConsensusConfig.java | 23 +++- .../iotdb/consensus/iot/IoTConsensus.java | 9 +- .../consensus/iot/IoTConsensusServerImpl.java | 79 +++++++++++- .../iot/client/DispatchLogHandler.java | 44 +++++++ .../DataNodeUserDataTransferAuditor.java | 70 ++++++++++ .../db/consensus/DataRegionConsensusImpl.java | 5 + .../IoTConsensusV2AsyncSink.java | 29 ++++- .../IoTConsensusV2SyncSink.java | 86 ++++++++++++ ...IoTConsensusV2TabletBatchEventHandler.java | 27 ++++ ...onsensusV2TabletInsertionEventHandler.java | 21 +++ ...onsensusV2TsFileInsertionEventHandler.java | 28 ++++ .../exchange/source/SourceHandle.java | 41 ++++++ .../plan/scheduler/AsyncPlanNodeSender.java | 58 ++++++++- .../scheduler/AsyncSendPlanNodeHandler.java | 68 +++++++++- .../FragmentInstanceDispatcherImpl.java | 66 +++++++++- .../load/LoadTsFileDispatcherImpl.java | 35 +++++ .../scheduler/AsyncPlanNodeSenderTest.java | 49 +++++++ .../iotdb/commons/i18n/CommonMessages.java | 5 + .../iotdb/commons/i18n/CommonMessages.java | 4 + .../commons/audit/AbstractAuditLogger.java | 51 ++++++++ .../iotdb/commons/audit/AuditEventType.java | 1 + .../audit/UserDataTransferAuditEvent.java | 122 ++++++++++++++++++ .../audit/UserDataTransferAuditHandler.java | 43 ++++++ .../audit/UserDataTransferErrorCode.java | 25 ++++ .../UserDataTransferProtectionMethod.java | 29 +++++ .../commons/audit/UserDataTransferType.java | 48 +++++++ .../audit/UserDataTransferAuditEventTest.java | 55 ++++++++ 27 files changed, 1107 insertions(+), 14 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java index 114a8aed4fcfb..4ce64d0ab9628 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.audit.TrustedChannelFailureHandler; +import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType; import java.util.List; @@ -39,6 +40,7 @@ public class ConsensusConfig { private final IoTConsensusV2Config iotConsensusV2Config; private final DirectoryStrategyType directoryStrategyType; private final TrustedChannelFailureHandler trustedChannelFailureHandler; + private final UserDataTransferAuditHandler userDataTransferAuditHandler; private ConsensusConfig( TEndPoint thisNode, @@ -50,7 +52,8 @@ private ConsensusConfig( IoTConsensusConfig iotConsensusConfig, IoTConsensusV2Config iotConsensusV2Config, DirectoryStrategyType directoryStrategyType, - TrustedChannelFailureHandler trustedChannelFailureHandler) { + TrustedChannelFailureHandler trustedChannelFailureHandler, + UserDataTransferAuditHandler userDataTransferAuditHandler) { this.thisNodeEndPoint = thisNode; this.thisNodeId = thisNodeId; this.storageDir = storageDir; @@ -61,6 +64,7 @@ private ConsensusConfig( this.iotConsensusV2Config = iotConsensusV2Config; this.directoryStrategyType = directoryStrategyType; this.trustedChannelFailureHandler = trustedChannelFailureHandler; + this.userDataTransferAuditHandler = userDataTransferAuditHandler; } public TEndPoint getThisNodeEndPoint() { @@ -103,6 +107,10 @@ public TrustedChannelFailureHandler getTrustedChannelFailureHandler() { return trustedChannelFailureHandler; } + public UserDataTransferAuditHandler getUserDataTransferAuditHandler() { + return userDataTransferAuditHandler; + } + public static ConsensusConfig.Builder newBuilder() { return new ConsensusConfig.Builder(); } @@ -121,6 +129,8 @@ public static class Builder { DirectoryStrategyType.MIN_FOLDER_OCCUPIED_SPACE_FIRST_STRATEGY; private TrustedChannelFailureHandler trustedChannelFailureHandler = TrustedChannelFailureHandler.NO_OP; + private UserDataTransferAuditHandler userDataTransferAuditHandler = + UserDataTransferAuditHandler.NO_OP; public ConsensusConfig build() { return new ConsensusConfig( @@ -135,7 +145,8 @@ public ConsensusConfig build() { Optional.ofNullable(iotConsensusV2Config) .orElseGet(() -> IoTConsensusV2Config.newBuilder().build()), directoryStrategyType, - trustedChannelFailureHandler); + trustedChannelFailureHandler, + userDataTransferAuditHandler); } public Builder setThisNode(TEndPoint thisNode) { @@ -190,5 +201,13 @@ public Builder setTrustedChannelFailureHandler( .orElse(TrustedChannelFailureHandler.NO_OP); return this; } + + public Builder setUserDataTransferAuditHandler( + UserDataTransferAuditHandler userDataTransferAuditHandler) { + this.userDataTransferAuditHandler = + Optional.ofNullable(userDataTransferAuditHandler) + .orElse(UserDataTransferAuditHandler.NO_OP); + return this; + } } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java index 477d8a5cb1175..0a4f70e4e3da4 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; @@ -104,6 +105,7 @@ public class IoTConsensus implements IConsensus { new ConcurrentHashMap<>(); private final IoTConsensusRPCService service; private final RegisterManager registerManager = new RegisterManager(); + private final UserDataTransferAuditHandler userDataTransferAuditHandler; private volatile IoTConsensusConfig config; /** @@ -131,6 +133,7 @@ public IoTConsensus(ConsensusConfig config, Registry registry) { this.recvSnapshotDirs = config.getRecvSnapshotDirs(); this.recvFolderStrategyType = config.getDirectoryStrategyType(); this.config = config.getIotConsensusConfig(); + this.userDataTransferAuditHandler = config.getUserDataTransferAuditHandler(); this.registry = registry; this.service = new IoTConsensusRPCService( @@ -207,7 +210,8 @@ private void initAndRecover() throws IOException { backgroundTaskService, clientManager, syncClientManager, - config); + config, + userDataTransferAuditHandler); stateMachineMap.put(consensusGroupId, consensus); } } catch (DiskSpaceInsufficientException e) { @@ -322,7 +326,8 @@ public void createLocalPeer(ConsensusGroupId groupId, List peers) backgroundTaskService, clientManager, syncClientManager, - config); + config, + userDataTransferAuditHandler); } catch (DiskSpaceInsufficientException e) { throw new RuntimeException(e); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index e074e7204ee51..f342079878bf9 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -21,6 +21,10 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; +import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; +import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.consensus.ConsensusGroupId; @@ -158,6 +162,7 @@ public class IoTConsensusServerImpl { private final ScheduledExecutorService backgroundTaskService; private final IoTConsensusRateLimiter ioTConsensusRateLimiter = IoTConsensusRateLimiter.getInstance(); + private final UserDataTransferAuditHandler userDataTransferAuditHandler; private IndexedConsensusRequest lastConsensusRequest; // Subscription queues receive IndexedConsensusRequest in real-time from write(), @@ -196,6 +201,33 @@ public IoTConsensusServerImpl( IClientManager syncClientManager, IoTConsensusConfig config) throws DiskSpaceInsufficientException { + this( + storageDir, + recvSnapshotDirs, + recvFolderStrategyType, + thisNode, + configuration, + stateMachine, + backgroundTaskService, + clientManager, + syncClientManager, + config, + UserDataTransferAuditHandler.NO_OP); + } + + public IoTConsensusServerImpl( + String storageDir, + List recvSnapshotDirs, + DirectoryStrategyType recvFolderStrategyType, + Peer thisNode, + Collection configuration, + IStateMachine stateMachine, + ScheduledExecutorService backgroundTaskService, + IClientManager clientManager, + IClientManager syncClientManager, + IoTConsensusConfig config, + UserDataTransferAuditHandler userDataTransferAuditHandler) + throws DiskSpaceInsufficientException { this.active = true; this.storageDir = storageDir; List snapshotDirs = new ArrayList<>(); @@ -215,6 +247,7 @@ public IoTConsensusServerImpl( this.configuration.addAll(configuration); this.backgroundTaskService = backgroundTaskService; this.config = config; + this.userDataTransferAuditHandler = userDataTransferAuditHandler; this.consensusGroupId = thisNode.getGroupId().toString(); this.consensusReqReader = (ConsensusReqReader) stateMachine.read(new GetConsensusReqReaderPlan()); @@ -433,7 +466,19 @@ public void transmitSnapshot(Peer targetPeer) throws ConsensusGroupModifyPeerExc TSendSnapshotFragmentReq req = reader.next().toTSendSnapshotFragmentReq(); req.setConsensusGroupId(targetPeer.getGroupId().convertToTConsensusGroupId()); ioTConsensusRateLimiter.acquireTransitDataSizeWithRateLimiter(req.getChunkLength()); - TSendSnapshotFragmentRes res = client.sendSnapshotFragment(req); + final TSendSnapshotFragmentRes res; + try { + res = client.sendSnapshotFragment(req); + recordSnapshotTransferAttempt( + targetPeer, + req, + isSuccess(res.getStatus()), + isSuccess(res.getStatus()) ? null : String.valueOf(res.getStatus().getCode()), + null); + } catch (Exception e) { + recordSnapshotTransferAttempt(targetPeer, req, false, null, e); + throw e; + } if (!isSuccess(res.getStatus())) { throw new ConsensusGroupModifyPeerException( String.format(IoTConsensusMessages.SNAPSHOT_TRANSMISSION_ERROR, targetPeer)); @@ -470,6 +515,34 @@ public void transmitSnapshot(Peer targetPeer) throws ConsensusGroupModifyPeerExc snapshotDir); } + private void recordSnapshotTransferAttempt( + Peer targetPeer, + TSendSnapshotFragmentReq request, + boolean success, + String errorCode, + Throwable error) { + if (!userDataTransferAuditHandler.isEnabled()) { + return; + } + try { + userDataTransferAuditHandler.onAttempt( + new UserDataTransferAuditEvent( + UserDataTransferType.IOT_CONSENSUS_SNAPSHOT, + thisNode.getEndpoint(), + thisNode.getEndpoint(), + targetPeer.getEndpoint(), + UserDataTransferProtectionMethod.fromTlsEnabled(config.getRpc().isEnableSSL()), + null, + request.getSnapshotId() + "/" + request.getOffset(), + 1, + success, + errorCode, + error)); + } catch (RuntimeException ignored) { + // Audit recording must not affect snapshot transmission. + } + } + public void receiveSnapshotFragment( String snapshotId, String originalFilePath, ByteBuffer fileChunk, long fileOffset) throws ConsensusGroupModifyPeerException { @@ -1121,6 +1194,10 @@ public Peer getThisNode() { return thisNode; } + public UserDataTransferAuditHandler getUserDataTransferAuditHandler() { + return userDataTransferAuditHandler; + } + public List getConfiguration() { List result = new ArrayList<>(configuration); Collections.sort(result); diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java index 64034144b9355..e80be25b4dec5 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java @@ -20,6 +20,9 @@ package org.apache.iotdb.consensus.iot.client; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; +import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.consensus.i18n.IoTConsensusMessages; import org.apache.iotdb.consensus.iot.logdispatcher.Batch; @@ -63,6 +66,15 @@ public DispatchLogHandler( @Override public void onComplete(TSyncLogEntriesRes response) { + final TSStatus failedStatus = + response.getStatuses().stream() + .filter(status -> status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .findFirst() + .orElse(null); + recordTransferAttempt( + failedStatus == null, + failedStatus == null ? null : String.valueOf(failedStatus.getCode()), + null); if (response.getStatuses().stream() .anyMatch(status -> RetryUtils.needRetryForWrite(status.getCode()))) { List retryStatusMessages = @@ -113,6 +125,7 @@ public void onComplete(TSyncLogEntriesRes response) { @Override public void onError(Exception exception) { + recordTransferAttempt(false, null, exception); ++retryCount; Throwable rootCause = ExceptionUtils.getRootCause(exception); final Throwable actualCause = rootCause == null ? exception : rootCause; @@ -172,4 +185,35 @@ private void completeBatch(Batch batch) { // removeBatch thread.updateSafelyDeletedSearchIndex(); } + + private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { + if (!thread.getImpl().getUserDataTransferAuditHandler().isEnabled()) { + return; + } + try { + thread + .getImpl() + .getUserDataTransferAuditHandler() + .onAttempt( + new UserDataTransferAuditEvent( + UserDataTransferType.IOT_CONSENSUS_LOG, + thread.getImpl().getThisNode().getEndpoint(), + thread.getImpl().getThisNode().getEndpoint(), + thread.getPeer().getEndpoint(), + UserDataTransferProtectionMethod.fromTlsEnabled( + thread.getConfig().getRpc().isEnableSSL()), + null, + thread.getImpl().getThisNode().getGroupId() + + "/" + + batch.getStartIndex() + + "-" + + batch.getEndIndex(), + retryCount + 1, + success, + errorCode, + error)); + } catch (RuntimeException ignored) { + // Audit recording must not affect consensus replication. + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java new file mode 100644 index 0000000000000..cc406bda68a03 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.audit; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; +import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; +import org.apache.iotdb.commons.audit.UserDataTransferType; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; + +import javax.annotation.Nullable; + +public final class DataNodeUserDataTransferAuditor { + + private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); + + private DataNodeUserDataTransferAuditor() {} + + public static boolean isEnabled() { + return COMMON_CONFIG.isEnableAuditLog(); + } + + public static void record( + UserDataTransferType transferType, + TEndPoint initiator, + TEndPoint source, + TEndPoint target, + @Nullable String context, + int attempt, + boolean success, + @Nullable String errorCode, + @Nullable Throwable error) { + if (!isEnabled()) { + return; + } + DNAuditLogger.getInstance() + .recordUserDataTransferAuditLog( + new UserDataTransferAuditEvent( + transferType, + initiator, + source, + target, + UserDataTransferProtectionMethod.fromTlsEnabled( + COMMON_CONFIG.isEnableInternalSSL()), + COMMON_CONFIG.isEnableInternalSSL() ? COMMON_CONFIG.getSslProtocol() : null, + context, + attempt, + success, + errorCode, + error)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index fbda403b320d7..0a16f643b6ace 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; @@ -143,6 +144,10 @@ private static ConsensusConfig buildConsensusConfig() { .setThisNode(new TEndPoint(CONF.getInternalAddress(), CONF.getDataRegionConsensusPort())) .setTrustedChannelFailureHandler( DNAuditLogger.getInstance()::recordTrustedChannelFailureAuditLogIfNecessary) + .setUserDataTransferAuditHandler( + COMMON_CONF.isEnableAuditLog() + ? DNAuditLogger.getInstance()::recordUserDataTransferAuditLog + : UserDataTransferAuditHandler.NO_OP) .setStorageDir(CONF.getDataRegionConsensusDir()) .setRecvSnapshotDirs(Arrays.asList(CONF.getLocalDataDirs())) // IoTConsensus always balances received snapshot files by least occupied space, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java index 05e14ed08ae8d..4faf31f3b702f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.consensus.ConsensusGroupId; @@ -37,6 +38,7 @@ import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName; import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeSink; import org.apache.iotdb.consensus.pipe.metric.IoTConsensusV2SyncLagManager; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodePipeMessages; @@ -740,12 +742,37 @@ private void logOnClientException( } } - private TEndPoint getFollowerUrl() { + public TEndPoint getFollowerUrl() { // In current iotConsensusV2 design, one connector corresponds to one follower, so the peers is // actually a singleton list return nodeUrls.get(0); } + public void recordUserDataTransferAudit( + UserDataTransferType transferType, + String context, + boolean success, + String errorCode, + Throwable error) { + if (!DataNodeUserDataTransferAuditor.isEnabled()) { + return; + } + final TEndPoint localEndPoint = + new TEndPoint( + IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), + IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusPort()); + DataNodeUserDataTransferAuditor.record( + transferType, + localEndPoint, + localEndPoint, + getFollowerUrl(), + context, + 1, + success, + errorCode, + error); + } + // synchronized to avoid close connector when transfer event @Override public synchronized void close() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java index 2f2741898094d..3641e6c9c1634 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.consensus.index.ProgressIndex; @@ -36,6 +37,8 @@ import org.apache.iotdb.consensus.iotconsensusv2.thrift.TIoTConsensusV2BatchTransferResp; import org.apache.iotdb.consensus.iotconsensusv2.thrift.TIoTConsensusV2TransferReq; import org.apache.iotdb.consensus.iotconsensusv2.thrift.TIoTConsensusV2TransferResp; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.consensus.metric.IoTConsensusV2SinkMetrics; import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; @@ -199,6 +202,7 @@ public void transfer(final Event event) throws Exception { } private void doTransfer() { + boolean transferAttemptRecorded = false; try (final SyncIoTConsensusV2ServiceClient syncIoTConsensusV2ServiceClient = syncRetryClientManager.borrowClient(getFollowerUrl())) { final TIoTConsensusV2BatchTransferResp resp; @@ -210,6 +214,15 @@ private void doTransfer() { resp.getBatchResps().stream() .map(TIoTConsensusV2TransferResp::getStatus) .collect(Collectors.toList()); + final TSStatus failedStatus = + statusList.stream().filter(status -> !isSuccessful(status)).findFirst().orElse(null); + recordTransferAttempt( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + String.valueOf(consensusGroupId), + failedStatus == null, + failedStatus == null ? null : String.valueOf(failedStatus.getCode()), + null); + transferAttemptRecorded = true; // TODO(support batch): handle retry logic // Only handle the failed statuses to avoid string format performance overhead @@ -225,6 +238,14 @@ private void doTransfer() { tabletBatchBuilder.onSuccess(); } catch (final Exception e) { + if (!transferAttemptRecorded) { + recordTransferAttempt( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + String.valueOf(consensusGroupId), + false, + null, + e); + } throw new PipeRuntimeSinkRetryTimesConfigurableException( String.format( IOT_CONSENSUS_V2_SYNC_CONNECTION_FAILED_FORMAT, @@ -332,6 +353,7 @@ private void doTransfer(PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletI pipeInsertNodeTabletInsertionEvent.getRebootTimes()); final TConsensusGroupId tConsensusGroupId = new TConsensusGroupId(TConsensusGroupType.DataRegion, consensusGroupId); + boolean transferAttemptRecorded = false; try (final SyncIoTConsensusV2ServiceClient syncIoTConsensusV2ServiceClient = syncRetryClientManager.borrowClient(getFollowerUrl())) { @@ -342,7 +364,23 @@ private void doTransfer(PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletI IoTConsensusV2TabletInsertNodeReq.toTIoTConsensusV2TransferReq( insertNode, tCommitId, tConsensusGroupId, progressIndex, thisDataNodeId); resp = syncIoTConsensusV2ServiceClient.iotConsensusV2Transfer(req); + final TSStatus status = resp.getStatus(); + recordTransferAttempt( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + String.valueOf(pipeInsertNodeTabletInsertionEvent.getReplicateIndexForIoTV2()), + isSuccessful(status), + isSuccessful(status) ? null : String.valueOf(status.getCode()), + null); + transferAttemptRecorded = true; } catch (final Exception e) { + if (!transferAttemptRecorded) { + recordTransferAttempt( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + String.valueOf(pipeInsertNodeTabletInsertionEvent.getReplicateIndexForIoTV2()), + false, + null, + e); + } throw new PipeRuntimeSinkRetryTimesConfigurableException( String.format( IOT_CONSENSUS_V2_SYNC_CONNECTION_FAILED_FORMAT, @@ -467,6 +505,8 @@ protected void transferFilePieces( ? readBuffer : Arrays.copyOfRange(readBuffer, 0, readLength); final IoTConsensusV2TransferFilePieceResp resp; + final long transferPosition = position; + boolean transferAttemptRecorded = false; try { resp = IoTConsensusV2TransferFilePieceResp.fromTIoTConsensusV2TransferResp( @@ -486,7 +526,23 @@ protected void transferFilePieces( tCommitId, tConsensusGroupId, thisDataNodeId))); + final TSStatus transferStatus = resp.getStatus(); + recordTransferAttempt( + UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, + file.getName() + "/" + transferPosition, + isSuccessful(transferStatus), + isSuccessful(transferStatus) ? null : String.valueOf(transferStatus.getCode()), + null); + transferAttemptRecorded = true; } catch (Exception e) { + if (!transferAttemptRecorded) { + recordTransferAttempt( + UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, + file.getName() + "/" + transferPosition, + false, + null, + e); + } throw new PipeRuntimeSinkRetryTimesConfigurableException( String.format( DataNodePipeMessages @@ -535,6 +591,36 @@ private TEndPoint getFollowerUrl() { return peers.get(0); } + private static boolean isSuccessful(TSStatus status) { + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || status.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(); + } + + private void recordTransferAttempt( + UserDataTransferType transferType, + String context, + boolean success, + String errorCode, + Throwable error) { + if (!DataNodeUserDataTransferAuditor.isEnabled()) { + return; + } + final TEndPoint localEndPoint = + new TEndPoint( + IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), + IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusPort()); + DataNodeUserDataTransferAuditor.record( + transferType, + localEndPoint, + localEndPoint, + getFollowerUrl(), + context, + 1, + success, + errorCode, + error); + } + // synchronized to avoid close connector when transfer event @Override public synchronized void close() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java index 54d8b3e6319b5..8b1cf0e19a8d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.handler; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -52,6 +53,7 @@ public class IoTConsensusV2TabletBatchEventHandler private final TIoTConsensusV2BatchTransferReq req; private final IoTConsensusV2AsyncSink connector; private final IoTConsensusV2SinkMetrics iotConsensusV2SinkMetrics; + private boolean transferAuditRecorded; public IoTConsensusV2TabletBatchEventHandler( final IoTConsensusV2AsyncBatchReqBuilder batchBuilder, @@ -84,6 +86,20 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { response.getBatchResps().stream() .map(TIoTConsensusV2TransferResp::getStatus) .collect(Collectors.toList()); + final TSStatus failedStatus = + status.stream() + .filter(tsStatus -> tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .findFirst() + .orElse(null); + connector.recordUserDataTransferAudit( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + requestCommitIds.isEmpty() + ? null + : requestCommitIds.get(0) + "/" + requestCommitIds.size(), + failedStatus == null, + failedStatus == null ? null : String.valueOf(failedStatus.getCode()), + null); + transferAuditRecorded = true; if (status.stream() .anyMatch( @@ -118,6 +134,17 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { @Override public void onError(final Exception exception) { + if (!transferAuditRecorded) { + connector.recordUserDataTransferAudit( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + requestCommitIds.isEmpty() + ? null + : requestCommitIds.get(0) + "/" + requestCommitIds.size(), + false, + null, + exception); + transferAuditRecorded = true; + } final Object pipeNames = events.stream() .map( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java index 3e1efd17088ee..347435fe366de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.handler; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -54,6 +55,7 @@ public abstract class IoTConsensusV2TabletInsertionEventHandler< protected final IoTConsensusV2SinkMetrics metric; private final long createTime; + private boolean transferAuditRecorded; protected IoTConsensusV2TabletInsertionEventHandler( TabletInsertionEvent event, @@ -83,6 +85,16 @@ public void onComplete(TIoTConsensusV2TransferResp response) { } final TSStatus status = response.getStatus(); + final boolean success = + status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || status.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(); + connector.recordUserDataTransferAudit( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + String.valueOf(((EnrichedEvent) event).getReplicateIndexForIoTV2()), + success, + success ? null : String.valueOf(status.getCode()), + null); + transferAuditRecorded = true; try { // Only handle the failed statuses to avoid string format performance overhead if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() @@ -113,6 +125,15 @@ public void onComplete(TIoTConsensusV2TransferResp response) { @Override public void onError(Exception exception) { + if (!transferAuditRecorded) { + connector.recordUserDataTransferAudit( + UserDataTransferType.IOT_CONSENSUS_V2_TABLET, + String.valueOf(((EnrichedEvent) event).getReplicateIndexForIoTV2()), + false, + null, + exception); + transferAuditRecorded = true; + } EnrichedEvent event = (EnrichedEvent) this.event; PipeLogger.log( ignored -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java index 8ebff37453196..9e561b6617f23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -87,6 +88,9 @@ public class IoTConsensusV2TsFileInsertionEventHandler private final long createTime; private long startTransferPieceTime; + private boolean currentAttemptContainsUserData; + private boolean transferAuditRecorded; + private String transferAuditContext; public IoTConsensusV2TsFileInsertionEventHandler( final PipeTsFileInsertionEvent event, @@ -161,6 +165,7 @@ public void transfer(final AsyncIoTConsensusV2ServiceClient client) transfer(client); } else if (currentFile == tsFile) { isSealSignalSent.set(true); + currentAttemptContainsUserData = false; client.iotConsensusV2Transfer( transferMod ? IoTConsensusV2TsFileSealWithModReq.toTIoTConsensusV2TransferReq( @@ -191,6 +196,9 @@ public void transfer(final AsyncIoTConsensusV2ServiceClient client) readLength == readFileBufferSize ? readBuffer : Arrays.copyOfRange(readBuffer, 0, readLength); + currentAttemptContainsUserData = true; + transferAuditRecorded = false; + transferAuditContext = currentFile.getName() + "/" + position; client.iotConsensusV2Transfer( transferMod ? IoTConsensusV2TsFilePieceWithModReq.toTIoTConsensusV2TransferReq( @@ -275,6 +283,17 @@ public void onComplete(final TIoTConsensusV2TransferResp response) { try { final IoTConsensusV2TransferFilePieceResp resp = IoTConsensusV2TransferFilePieceResp.fromTIoTConsensusV2TransferResp(response); + final TSStatus transferStatus = resp.getStatus(); + final boolean success = + transferStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || transferStatus.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(); + connector.recordUserDataTransferAudit( + UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, + transferAuditContext, + success, + success ? null : String.valueOf(transferStatus.getCode()), + null); + transferAuditRecorded = true; // This case only happens when the connection is broken, and the connector is reconnected // to the receiver, then the receiver will redirect the file position to the last position @@ -309,6 +328,15 @@ public void onComplete(final TIoTConsensusV2TransferResp response) { @Override public void onError(final Exception exception) { + if (currentAttemptContainsUserData && !transferAuditRecorded) { + connector.recordUserDataTransferAudit( + UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, + transferAuditContext, + false, + null, + exception); + transferAuditRecorded = true; + } PipeLogger.log( ignored -> LOGGER.warn( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java index 2cd33b2d769a6..d3ca71b89ad37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java @@ -20,9 +20,12 @@ package org.apache.iotdb.db.queryengine.execution.exchange.source; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserDataTransferErrorCode; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; @@ -73,6 +76,7 @@ public class SourceHandle implements ISourceHandle { private static final long DEFAULT_RETRY_INTERVAL_IN_MS = 1000; private final TEndPoint remoteEndpoint; + private final TEndPoint localEndpoint; private final TFragmentInstanceId remoteFragmentInstanceId; private final TFragmentInstanceId localFragmentInstanceId; @@ -175,6 +179,10 @@ public SourceHandle( Validate.notNull( remoteEndpoint, DataNodeQueryMessages.EXCEPTION_REMOTEENDPOINT_CAN_NOT_BE_NULL_DOT_DE2B5885); + this.localEndpoint = + new TEndPoint( + IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), + IoTDBDescriptor.getInstance().getConfig().getMppDataExchangePort()); this.remoteFragmentInstanceId = Validate.notNull( remoteFragmentInstanceId, @@ -637,11 +645,15 @@ public void run() { attempt += 1; long startTime = System.nanoTime(); + boolean transferAttemptRecorded = false; try (SyncDataNodeMPPDataExchangeServiceClient client = mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint)) { TGetDataBlockResponse resp = client.getDataBlock(req); int tsBlockNum = resp.getTsBlocks().size(); if (tsBlockNum == 0) { + recordTransferAttempt( + attempt, false, UserDataTransferErrorCode.EMPTY_RESPONSE.name(), null); + transferAttemptRecorded = true; if (!closed) { // failed to pull TsBlocks LOGGER.warn( @@ -654,6 +666,8 @@ public void run() { } return; } + recordTransferAttempt(attempt, true, null, null); + transferAttemptRecorded = true; List tsBlocks = new ArrayList<>(tsBlockNum); tsBlocks.addAll(resp.getTsBlocks()); @@ -681,6 +695,10 @@ public void run() { break; } catch (Throwable e) { + if (!transferAttemptRecorded) { + recordTransferAttempt(attempt, false, null, e); + } + LOGGER.warn( DataNodeQueryMessages.FAILED_TO_GET_DATA_BLOCK, startSequenceId, @@ -710,6 +728,29 @@ public void run() { } } + private void recordTransferAttempt( + int attempt, boolean success, String errorCode, Throwable error) { + if (!DataNodeUserDataTransferAuditor.isEnabled()) { + return; + } + DataNodeUserDataTransferAuditor.record( + UserDataTransferType.MPP_TS_BLOCK, + localEndpoint, + remoteEndpoint, + localEndpoint, + remoteFragmentInstanceId + + "/" + + indexOfUpstreamSinkHandle + + "/" + + startSequenceId + + "-" + + endSequenceId, + attempt, + success, + errorCode, + error); + } + private void fail(Throwable t) { synchronized (SourceHandle.this) { if (aborted || closed) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java index da5a2dfc889b1..31bdae0728fea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java @@ -23,8 +23,11 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.mpp.rpc.thrift.TPlanNode; import org.apache.iotdb.mpp.rpc.thrift.TSendBatchPlanNodeReq; import org.apache.iotdb.mpp.rpc.thrift.TSendSinglePlanNodeReq; @@ -49,6 +52,7 @@ public class AsyncPlanNodeSender { private final IClientManager asyncInternalServiceClientManager; private final List instances; + private final TEndPoint localEndPoint; private final Map batchRequests; private final Map instanceId2RespMap; @@ -57,6 +61,7 @@ public class AsyncPlanNodeSender { private final AtomicLong pendingNumber; private long startSendTime; + private int auditAttempt; public AsyncPlanNodeSender( IClientManager @@ -65,6 +70,10 @@ public AsyncPlanNodeSender( this.startSendTime = System.nanoTime(); this.asyncInternalServiceClientManager = asyncInternalServiceClientManager; this.instances = instances; + this.localEndPoint = + new TEndPoint( + IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), + IoTDBDescriptor.getInstance().getConfig().getInternalPort()); this.batchRequests = new HashMap<>(); for (int i = 0; i < instances.size(); i++) { this.batchRequests @@ -76,7 +85,8 @@ public AsyncPlanNodeSender( new TSendSinglePlanNodeReq( new TPlanNode( instances.get(i).getFragment().getPlanNodeTree().serializeToByteBuffer()), - instances.get(i).getRegionReplicaSet().getRegionId())); + instances.get(i).getRegionReplicaSet().getRegionId()), + containsInsertNode(instances.get(i).getFragment().getPlanNodeTree())); } this.instanceId2RespMap = new ConcurrentHashMap<>(instances.size() + 1, 1); this.needRetryInstanceIndex = Collections.synchronizedList(new ArrayList<>()); @@ -84,6 +94,7 @@ public AsyncPlanNodeSender( } public void sendAll() { + auditAttempt++; for (Map.Entry entry : batchRequests.entrySet()) { AsyncSendPlanNodeHandler handler = new AsyncSendPlanNodeHandler( @@ -91,7 +102,12 @@ public void sendAll() { pendingNumber, instanceId2RespMap, needRetryInstanceIndex, - startSendTime); + startSendTime, + localEndPoint, + entry.getKey(), + entry.getValue().containsUserData(), + buildAuditContext(entry.getValue()), + auditAttempt); try { AsyncDataNodeInternalServiceClient client = asyncInternalServiceClientManager.borrowClient(entry.getKey()); @@ -188,7 +204,9 @@ public void retry() throws InterruptedException { .getFragment() .getPlanNodeTree() .serializeToByteBuffer()), - instances.get(fragmentInstanceIndex).getRegionReplicaSet().getRegionId())); + instances.get(fragmentInstanceIndex).getRegionReplicaSet().getRegionId()), + containsInsertNode( + instances.get(fragmentInstanceIndex).getFragment().getPlanNodeTree())); } // 2. reset the pendingNumber, needRetryInstanceIds and startSendTime @@ -213,9 +231,13 @@ static class BatchRequestWithIndex { private final List indexes = new ArrayList<>(); private final TSendBatchPlanNodeReq batchRequest = new TSendBatchPlanNodeReq(); - void addSinglePlanNodeReq(int index, TSendSinglePlanNodeReq singleRequest) { + private boolean containsUserData; + + void addSinglePlanNodeReq( + int index, TSendSinglePlanNodeReq singleRequest, boolean containsUserData) { indexes.add(index); batchRequest.addToRequests(singleRequest); + this.containsUserData |= containsUserData; } public List getIndexes() { @@ -225,5 +247,33 @@ public List getIndexes() { public TSendBatchPlanNodeReq getBatchRequest() { return batchRequest; } + + public boolean containsUserData() { + return containsUserData; + } + } + + private String buildAuditContext(BatchRequestWithIndex batchRequest) { + if (batchRequest.getIndexes().isEmpty()) { + return null; + } + return instances.get(batchRequest.getIndexes().get(0)).getId().getFullId() + + "/" + + batchRequest.getIndexes().size(); + } + + static boolean containsInsertNode(PlanNode node) { + if (node instanceof InsertNode) { + return true; + } + if (node.getChildren() == null) { + return false; + } + for (PlanNode child : node.getChildren()) { + if (containsInsertNode(child)) { + return true; + } + } + return false; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java index 23f99da3484c2..fe9ce662a3fd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java @@ -19,8 +19,12 @@ package org.apache.iotdb.db.queryengine.plan.scheduler; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserDataTransferErrorCode; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.StatusUtils; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.mpp.rpc.thrift.TSendBatchPlanNodeResp; import org.apache.iotdb.mpp.rpc.thrift.TSendSinglePlanNodeResp; import org.apache.iotdb.rpc.RpcUtils; @@ -43,6 +47,12 @@ public class AsyncSendPlanNodeHandler implements AsyncMethodCallback instanceId2RespMap; private final List needRetryInstanceIndex; private final long sendTime; + private final TEndPoint localEndPoint; + private final TEndPoint targetEndPoint; + private final boolean containsUserData; + private final String auditContext; + private final int auditAttempt; + private boolean transferAuditRecorded; private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = PerformanceOverviewMetrics.getInstance(); @@ -51,16 +61,27 @@ public AsyncSendPlanNodeHandler( AtomicLong pendingNumber, Map instanceId2RespMap, List needRetryInstanceIndex, - long sendTime) { + long sendTime, + TEndPoint localEndPoint, + TEndPoint targetEndPoint, + boolean containsUserData, + String auditContext, + int auditAttempt) { this.instanceIds = instanceIds; this.pendingNumber = pendingNumber; this.instanceId2RespMap = instanceId2RespMap; this.needRetryInstanceIndex = needRetryInstanceIndex; this.sendTime = sendTime; + this.localEndPoint = localEndPoint; + this.targetEndPoint = targetEndPoint; + this.containsUserData = containsUserData; + this.auditContext = auditContext; + this.auditAttempt = auditAttempt; } @Override public void onComplete(TSendBatchPlanNodeResp sendBatchPlanNodeResp) { + recordTransferAttempt(sendBatchPlanNodeResp); for (int i = 0; i < sendBatchPlanNodeResp.getResponses().size(); i++) { TSendSinglePlanNodeResp singlePlanNodeResp = sendBatchPlanNodeResp.getResponses().get(i); instanceId2RespMap.put(instanceIds.get(i), singlePlanNodeResp); @@ -78,6 +99,9 @@ public void onComplete(TSendBatchPlanNodeResp sendBatchPlanNodeResp) { @Override public void onError(Exception e) { + if (!transferAuditRecorded) { + recordTransferAttempt(false, null, e); + } if (needRetry(e)) { needRetryInstanceIndex.addAll(instanceIds); } @@ -106,4 +130,46 @@ public static boolean needRetry(Exception e) { private boolean needRetry(TSendSinglePlanNodeResp resp) { return !resp.accepted && resp.status != null && StatusUtils.needRetryHelper(resp.status); } + + private void recordTransferAttempt(TSendBatchPlanNodeResp response) { + if (!containsUserData) { + return; + } + if (response.getResponsesSize() != instanceIds.size()) { + recordTransferAttempt(false, UserDataTransferErrorCode.REMOTE_REJECTED.name(), null); + return; + } + for (TSendSinglePlanNodeResp singleResponse : response.getResponses()) { + if (!singleResponse.isAccepted() + || (singleResponse.isSetStatus() + && singleResponse.getStatus().getCode() + != TSStatusCode.SUCCESS_STATUS.getStatusCode())) { + recordTransferAttempt( + false, + singleResponse.isSetStatus() + ? String.valueOf(singleResponse.getStatus().getCode()) + : UserDataTransferErrorCode.REMOTE_REJECTED.name(), + null); + return; + } + } + recordTransferAttempt(true, null, null); + } + + private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { + if (!containsUserData || !DataNodeUserDataTransferAuditor.isEnabled()) { + return; + } + DataNodeUserDataTransferAuditor.record( + UserDataTransferType.INSERT_PLAN_NODE, + localEndPoint, + localEndPoint, + targetEndPoint, + auditContext, + auditAttempt, + success, + errorCode, + error); + transferAuditRecorded = true; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 7ae9d8d6f77d7..d122681ce7717 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -23,6 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserDataTransferErrorCode; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -35,6 +37,7 @@ import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException; import org.apache.iotdb.consensus.exception.RatisReadUnavailableException; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; import org.apache.iotdb.db.exception.query.QueryTimeoutRuntimeException; @@ -465,12 +468,17 @@ private boolean isDispatchedToLocal(TEndPoint endPoint) { return this.localhostIpAddr.equals(endPoint.getIp()) && localhostInternalPort == endPoint.port; } - private void dispatchRemoteHelper(final FragmentInstance instance, final TEndPoint endPoint) + private void dispatchRemoteHelper( + final FragmentInstance instance, final TEndPoint endPoint, final int attempt) throws FragmentInstanceDispatchException, TException, ClientManagerException, RatisReadUnavailableException, ConsensusGroupNotExistException { + final boolean containsUserData = + (instance.getType() == QueryType.WRITE || instance.getType() == QueryType.OTHER) + && AsyncPlanNodeSender.containsInsertNode(instance.getFragment().getPlanNodeTree()); + boolean transferAttemptRecorded = false; try (final SyncDataNodeInternalServiceClient client = syncInternalServiceClientManager.borrowClient(endPoint)) { switch (instance.getType()) { @@ -523,6 +531,28 @@ private void dispatchRemoteHelper(final FragmentInstance instance, final TEndPoi instance.getRegionReplicaSet().getRegionId()))); final TSendSinglePlanNodeResp sendPlanNodeResp = client.sendBatchPlanNode(sendPlanNodeReq).getResponses().get(0); + if (containsUserData && DataNodeUserDataTransferAuditor.isEnabled()) { + final boolean success = + sendPlanNodeResp.isAccepted() + && (!sendPlanNodeResp.isSetStatus() + || sendPlanNodeResp.getStatus().getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode()); + DataNodeUserDataTransferAuditor.record( + UserDataTransferType.INSERT_PLAN_NODE, + new TEndPoint(localhostIpAddr, localhostInternalPort), + new TEndPoint(localhostIpAddr, localhostInternalPort), + endPoint, + instance.getId().getFullId(), + attempt, + success, + success + ? null + : sendPlanNodeResp.isSetStatus() + ? String.valueOf(sendPlanNodeResp.getStatus().getCode()) + : UserDataTransferErrorCode.REMOTE_REJECTED.name(), + null); + transferAttemptRecorded = true; + } if (!sendPlanNodeResp.accepted) { if (sendPlanNodeResp.getStatus() == null) { throw new FragmentInstanceDispatchException( @@ -556,7 +586,15 @@ private void dispatchRemoteHelper(final FragmentInstance instance, final TEndPoi TSStatusCode.EXECUTE_STATEMENT_ERROR, String.format(DataNodeQueryMessages.UNKNOWN_READ_TYPE_FMT, instance.getType()))); } + } catch (ClientManagerException e) { + if (!transferAttemptRecorded) { + recordWriteTransferFailureIfNecessary(instance, endPoint, attempt, containsUserData, e); + } + throw e; } catch (TException e) { + if (!transferAttemptRecorded) { + recordWriteTransferFailureIfNecessary(instance, endPoint, attempt, containsUserData, e); + } Throwable rootCause = ExceptionUtils.getRootCause(e); if (rootCause instanceof TTransportException && ((TTransportException) rootCause).getType() == TTransportException.CORRUPTED_DATA) { @@ -568,6 +606,28 @@ private void dispatchRemoteHelper(final FragmentInstance instance, final TEndPoi } } + private void recordWriteTransferFailureIfNecessary( + FragmentInstance instance, + TEndPoint endPoint, + int attempt, + boolean containsUserData, + Throwable error) { + if (!containsUserData || !DataNodeUserDataTransferAuditor.isEnabled()) { + return; + } + final TEndPoint localEndPoint = new TEndPoint(localhostIpAddr, localhostInternalPort); + DataNodeUserDataTransferAuditor.record( + UserDataTransferType.INSERT_PLAN_NODE, + localEndPoint, + localEndPoint, + endPoint, + instance.getId().getFullId(), + attempt, + false, + null, + error); + } + private void dispatchRemoteFailed(TEndPoint endPoint, Exception e) throws FragmentInstanceDispatchException { LOGGER.warn( @@ -587,7 +647,7 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) throws FragmentInstanceDispatchException { try { - dispatchRemoteHelper(instance, endPoint); + dispatchRemoteHelper(instance, endPoint, 1); } catch (ClientManagerException | TException | RatisReadUnavailableException e) { LOGGER.warn( DataNodeQueryMessages @@ -610,7 +670,7 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) } // we just retry once to clear stale connection for a restart node. try { - dispatchRemoteHelper(instance, endPoint); + dispatchRemoteHelper(instance, endPoint, 2); } catch (ClientManagerException | TException | RatisReadUnavailableException diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java index 3f5020cccf31e..53d8310e3a7c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -24,6 +24,8 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.audit.UserDataTransferErrorCode; +import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; @@ -32,6 +34,7 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.ProgressIndexType; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadFileException; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; @@ -222,16 +225,30 @@ public void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDi private void dispatchRemote(TTsFilePieceReq loadTsFileReq, TEndPoint endPoint) throws FragmentInstanceDispatchException { + boolean transferAttemptRecorded = false; try (SyncDataNodeInternalServiceClient client = internalServiceClientManager.borrowClient(endPoint)) { client.setTimeout(CONNECTION_TIMEOUT_MS.get()); final TLoadResp loadResp = client.sendTsFilePieceNode(loadTsFileReq); if (!loadResp.isAccepted()) { + recordTransferAttempt( + endPoint, + false, + loadResp.isSetStatus() + ? String.valueOf(loadResp.getStatus().getCode()) + : UserDataTransferErrorCode.REMOTE_REJECTED.name(), + null); + transferAttemptRecorded = true; LOGGER.warn(loadResp.message); throw new FragmentInstanceDispatchException(loadResp.status); } + recordTransferAttempt(endPoint, true, null, null); + transferAttemptRecorded = true; } catch (Exception e) { + if (!transferAttemptRecorded) { + recordTransferAttempt(endPoint, false, null, e); + } adjustTimeoutIfNecessary(e); final String exceptionMessage = @@ -246,6 +263,24 @@ private void dispatchRemote(TTsFilePieceReq loadTsFileReq, TEndPoint endPoint) } } + private void recordTransferAttempt( + TEndPoint target, boolean success, String errorCode, Throwable error) { + if (!DataNodeUserDataTransferAuditor.isEnabled()) { + return; + } + final TEndPoint localEndPoint = new TEndPoint(localhostIpAddr, localhostInternalPort); + DataNodeUserDataTransferAuditor.record( + UserDataTransferType.LOAD_TSFILE_PIECE, + localEndPoint, + localEndPoint, + target, + uuid, + 1, + success, + errorCode, + error); + } + public Future dispatchCommand( TLoadCommandReq originalLoadCommandReq, Set replicaSets) { Set allEndPoint = new HashSet<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java new file mode 100644 index 0000000000000..ff4ddb27354fa --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.scheduler; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; + +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AsyncPlanNodeSenderTest { + + @Test + public void testOnlyInsertPayloadIsClassifiedAsUserData() { + final PlanNode queryPlan = mock(PlanNode.class); + when(queryPlan.getChildren()).thenReturn(Collections.emptyList()); + assertFalse(AsyncPlanNodeSender.containsInsertNode(queryPlan)); + + final InsertNode insertNode = mock(InsertNode.class); + assertTrue(AsyncPlanNodeSender.containsInsertNode(insertNode)); + + final PlanNode wrapper = mock(PlanNode.class); + when(wrapper.getChildren()).thenReturn(Collections.singletonList(insertNode)); + assertTrue(AsyncPlanNodeSender.containsInsertNode(wrapper)); + } +} diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index 94194a086b47c..9181235b1cb33 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -224,5 +224,10 @@ private CommonMessages() {} public static final String EXCEPTION_DISK_SPACE_WARNING_THRESHOLD_MUST_BE_IN_0_1_BUT_WAS_7B345766 = "disk_space_warning_threshold must be in [0, 1), but was "; public static final String LOG_TRUSTED_CHANNEL_FUNCTION_FAILED_INITIATOR_ARG_TARGET_ARG_E4C28443 = "Trusted channel function failed: initiator=%s, target=%s"; + public static final String + LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_TYPE_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_PROTECTION_PROTOCOL_ARG_CONTEXT_ARG_ATTEMPT_ARG_ERROR_CODE_ARG_ERROR_TYPE_ARG_941238A8 = + "User data transfer attempt: time=%d, type=%s, initiator=%s, source=%s, target=%s," + + " protection_method=%s, protection_protocol=%s, context=%s, attempt=%d," + + " error_code=%s, error_type=%s"; } diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index 07d886a44f3d4..0353c395700a0 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -217,5 +217,9 @@ private CommonMessages() {} public static final String EXCEPTION_DISK_SPACE_WARNING_THRESHOLD_MUST_BE_IN_0_1_BUT_WAS_7B345766 = "disk_space_warning_threshold 必须在 [0, 1) 范围内，但实际为 "; public static final String LOG_TRUSTED_CHANNEL_FUNCTION_FAILED_INITIATOR_ARG_TARGET_ARG_E4C28443 = "可信信道功能失效：发起者=%s，目标端=%s"; + public static final String + LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_TYPE_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_PROTECTION_PROTOCOL_ARG_CONTEXT_ARG_ATTEMPT_ARG_ERROR_CODE_ARG_ERROR_TYPE_ARG_941238A8 = + "用户数据传送尝试：时间=%d，类型=%s，发起者=%s，源端=%s，目标端=%s，保护方法=%s，保护协议=%s，" + + "上下文=%s，尝试次数=%d，错误码=%s，错误类型=%s"; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java index d4b94265f2374..7af0c31cfe159 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java @@ -35,6 +35,8 @@ public abstract class AbstractAuditLogger { private static final long INTERNAL_AUDIT_LOG_USER_ID = 4; private static final ThreadLocal RECORDING_TRUSTED_CHANNEL_FAILURE = ThreadLocal.withInitial(() -> false); + private static final ThreadLocal RECORDING_USER_DATA_TRANSFER = + ThreadLocal.withInitial(() -> false); public static final String OBJECT_AUTHENTICATION_AUDIT_STR = "User %s (ID=%d) requests authority on object %s with result %s"; @@ -136,4 +138,53 @@ public void recordTrustedChannelFailureAuditLogIfNecessary( RECORDING_TRUSTED_CHANNEL_FAILURE.remove(); } } + + /** Records one user-data transfer attempt without retaining any transferred payload. */ + public void recordUserDataTransferAuditLog(UserDataTransferAuditEvent event) { + if (!IS_AUDIT_LOG_ENABLED + || event == null + || event.getInitiator() == null + || event.getSource() == null + || event.getTarget() == null + || Boolean.TRUE.equals(RECORDING_USER_DATA_TRANSFER.get())) { + return; + } + + final String initiatorIdentifier = NodeUrlUtils.convertTEndPointUrl(event.getInitiator()); + final String sourceIdentifier = NodeUrlUtils.convertTEndPointUrl(event.getSource()); + final String targetIdentifier = NodeUrlUtils.convertTEndPointUrl(event.getTarget()); + RECORDING_USER_DATA_TRANSFER.set(true); + try { + log( + new AuditLogFields( + INTERNAL_AUDIT_LOG_USER_ID, + User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME, + initiatorIdentifier, + AuditEventType.USER_DATA_TRANSFER, + event.getTransferType().getOperation(), + event.getTransferType().getPrivilegeType(), + event.isSuccess(), + null, + null), + () -> + String.format( + CommonMessages + .LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_TYPE_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_PROTECTION_PROTOCOL_ARG_CONTEXT_ARG_ATTEMPT_ARG_ERROR_CODE_ARG_ERROR_TYPE_ARG_941238A8, + event.getTimestamp(), + event.getTransferType(), + initiatorIdentifier, + sourceIdentifier, + targetIdentifier, + event.getProtectionMethod(), + event.getProtectionProtocol(), + event.getContext(), + event.getAttempt(), + event.getErrorCode(), + event.getErrorType())); + } catch (RuntimeException ignored) { + // Audit recording must not affect the user-data transfer being audited. + } finally { + RECORDING_USER_DATA_TRANSFER.remove(); + } + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AuditEventType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AuditEventType.java index d9e2a1f5c304f..d1f8e63058a38 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AuditEventType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AuditEventType.java @@ -46,6 +46,7 @@ public enum AuditEventType { SESSION_TIME_EXCEEDED, LOGIN_REJECT_IP, TRUSTED_CHANNEL_FUNCTION_FAILURE, + USER_DATA_TRANSFER, SYSTEM_OPERATION, DN_SHUTDOWN; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java new file mode 100644 index 0000000000000..f934bec22b988 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.audit; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; + +import javax.annotation.Nullable; + +/** + * Describes one attempt to transfer user data between physically separated parts of IoTDB. Payload + * contents and exception messages must not be included in this event. + */ +public final class UserDataTransferAuditEvent { + + private final long timestamp; + private final UserDataTransferType transferType; + private final TEndPoint initiator; + private final TEndPoint source; + private final TEndPoint target; + private final UserDataTransferProtectionMethod protectionMethod; + private final String protectionProtocol; + private final String context; + private final int attempt; + private final boolean success; + private final String errorCode; + private final String errorType; + + public UserDataTransferAuditEvent( + UserDataTransferType transferType, + TEndPoint initiator, + TEndPoint source, + TEndPoint target, + UserDataTransferProtectionMethod protectionMethod, + @Nullable String protectionProtocol, + @Nullable String context, + int attempt, + boolean success, + @Nullable String errorCode, + @Nullable Throwable error) { + this.timestamp = System.currentTimeMillis(); + this.transferType = transferType; + this.initiator = initiator; + this.source = source; + this.target = target; + this.protectionMethod = protectionMethod; + this.protectionProtocol = protectionProtocol; + this.context = context; + this.attempt = attempt; + this.success = success; + this.errorCode = errorCode; + this.errorType = error == null ? null : error.getClass().getName(); + } + + public long getTimestamp() { + return timestamp; + } + + public UserDataTransferType getTransferType() { + return transferType; + } + + public TEndPoint getInitiator() { + return initiator; + } + + public TEndPoint getSource() { + return source; + } + + public TEndPoint getTarget() { + return target; + } + + public UserDataTransferProtectionMethod getProtectionMethod() { + return protectionMethod; + } + + @Nullable + public String getProtectionProtocol() { + return protectionProtocol; + } + + @Nullable + public String getContext() { + return context; + } + + public int getAttempt() { + return attempt; + } + + public boolean isSuccess() { + return success; + } + + @Nullable + public String getErrorCode() { + return errorCode; + } + + @Nullable + public String getErrorType() { + return errorType; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java new file mode 100644 index 0000000000000..a5e0619b741a3 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.audit; + +@FunctionalInterface +public interface UserDataTransferAuditHandler { + + UserDataTransferAuditHandler NO_OP = + new UserDataTransferAuditHandler() { + @Override + public void onAttempt(UserDataTransferAuditEvent event) { + // Do nothing. + } + + @Override + public boolean isEnabled() { + return false; + } + }; + + void onAttempt(UserDataTransferAuditEvent event); + + default boolean isEnabled() { + return true; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java new file mode 100644 index 0000000000000..70e461887df42 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.audit; + +public enum UserDataTransferErrorCode { + EMPTY_RESPONSE, + REMOTE_REJECTED +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java new file mode 100644 index 0000000000000..304e3d8d221f3 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.audit; + +public enum UserDataTransferProtectionMethod { + TLS, + UNPROTECTED; + + public static UserDataTransferProtectionMethod fromTlsEnabled(boolean tlsEnabled) { + return tlsEnabled ? TLS : UNPROTECTED; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java new file mode 100644 index 0000000000000..9219b6a71b5b6 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.audit; + +import org.apache.iotdb.commons.auth.entity.PrivilegeType; + +public enum UserDataTransferType { + MPP_TS_BLOCK(AuditLogOperation.QUERY, PrivilegeType.READ_DATA), + INSERT_PLAN_NODE(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), + LOAD_TSFILE_PIECE(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), + IOT_CONSENSUS_LOG(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), + IOT_CONSENSUS_SNAPSHOT(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), + IOT_CONSENSUS_V2_TABLET(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), + IOT_CONSENSUS_V2_TSFILE(AuditLogOperation.DML, PrivilegeType.WRITE_DATA); + + private final AuditLogOperation operation; + private final PrivilegeType privilegeType; + + UserDataTransferType(AuditLogOperation operation, PrivilegeType privilegeType) { + this.operation = operation; + this.privilegeType = privilegeType; + } + + public AuditLogOperation getOperation() { + return operation; + } + + public PrivilegeType getPrivilegeType() { + return privilegeType; + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java new file mode 100644 index 0000000000000..47c392ea9cadc --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.audit; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; + +public class UserDataTransferAuditEventTest { + + @Test + public void testRecordsErrorTypeWithoutErrorMessage() { + final UserDataTransferAuditEvent event = + new UserDataTransferAuditEvent( + UserDataTransferType.MPP_TS_BLOCK, + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.2", 10740), + new TEndPoint("127.0.0.1", 10740), + UserDataTransferProtectionMethod.TLS, + "TLSv1.3", + "query/0-1", + 2, + false, + null, + new IOException("payload must not be retained")); + + assertEquals(IOException.class.getName(), event.getErrorType()); + assertNull(event.getErrorCode()); + assertEquals(2, event.getAttempt()); + assertFalse(event.isSuccess()); + } +} From 43a7104b179e07dfb3c8040b68ccf40baa34facb Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 2 Sep 2026 15:25:52 +0800 Subject: [PATCH 2/8] [Feature] Address review for user data transfer audit hooks --- .../request/IndexedConsensusRequest.java | 10 ++ .../consensus/config/ConsensusConfig.java | 22 ++- .../UserDataTransferAuditClassifier.java | 31 ++++ .../iotdb/consensus/iot/IoTConsensus.java | 9 +- .../consensus/iot/IoTConsensusServerImpl.java | 38 +++-- .../iot/client/DispatchLogHandler.java | 61 +++++--- .../consensus/iot/logdispatcher/Batch.java | 10 ++ .../iot/logdispatcher/LogDispatcher.java | 5 +- .../iot/client/DispatchLogHandlerTest.java | 140 ++++++++++++++++++ .../DataNodeUserDataTransferAuditor.java | 98 +++++++++--- .../db/consensus/DataRegionConsensusImpl.java | 2 + .../IoTConsensusV2AsyncSink.java | 21 +-- .../IoTConsensusV2SyncSink.java | 48 +----- ...IoTConsensusV2TabletBatchEventHandler.java | 14 +- ...onsensusV2TabletInsertionEventHandler.java | 14 +- ...onsensusV2TsFileInsertionEventHandler.java | 16 +- .../exchange/source/SourceHandle.java | 65 ++++---- .../plan/scheduler/AsyncPlanNodeSender.java | 43 ++---- .../scheduler/AsyncSendPlanNodeHandler.java | 21 +-- .../FragmentInstanceDispatcherImpl.java | 39 ++--- .../load/LoadTsFileDispatcherImpl.java | 14 +- .../DataNodeUserDataTransferAuditorTest.java | 52 +++++++ .../scheduler/AsyncPlanNodeSenderTest.java | 4 + .../iotdb/commons/i18n/CommonMessages.java | 7 +- .../iotdb/commons/i18n/CommonMessages.java | 5 +- .../commons/audit/AbstractAuditLogger.java | 15 +- .../audit/UserDataTransferAuditEvent.java | 48 +----- .../audit/UserDataTransferAuditHandler.java | 5 + .../audit/UserDataTransferErrorCode.java | 2 + .../commons/audit/UserDataTransferType.java | 48 ------ .../audit/UserDataTransferAuditEventTest.java | 14 +- 31 files changed, 515 insertions(+), 406 deletions(-) create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/request/IndexedConsensusRequest.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/request/IndexedConsensusRequest.java index 834a752be6a3d..6f67c10d535aa 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/request/IndexedConsensusRequest.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/request/IndexedConsensusRequest.java @@ -49,6 +49,7 @@ public class IndexedConsensusRequest implements IConsensusRequest { private long memorySize = 0; private long retainedMemorySize = 0; private boolean serializedRequestsBuilt = false; + private boolean containsUserData = false; private final AtomicLong referenceCnt = new AtomicLong(); public IndexedConsensusRequest(long searchIndex, List requests) { @@ -171,6 +172,15 @@ public IndexedConsensusRequest setNodeId(int nodeId) { return this; } + public boolean containsUserData() { + return containsUserData; + } + + public IndexedConsensusRequest setContainsUserData(boolean containsUserData) { + this.containsUserData = containsUserData; + return this; + } + public long getLocalSeq() { return searchIndex; } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java index 4ce64d0ab9628..e3ea222088ccb 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/ConsensusConfig.java @@ -41,6 +41,7 @@ public class ConsensusConfig { private final DirectoryStrategyType directoryStrategyType; private final TrustedChannelFailureHandler trustedChannelFailureHandler; private final UserDataTransferAuditHandler userDataTransferAuditHandler; + private final UserDataTransferAuditClassifier userDataTransferAuditClassifier; private ConsensusConfig( TEndPoint thisNode, @@ -53,7 +54,8 @@ private ConsensusConfig( IoTConsensusV2Config iotConsensusV2Config, DirectoryStrategyType directoryStrategyType, TrustedChannelFailureHandler trustedChannelFailureHandler, - UserDataTransferAuditHandler userDataTransferAuditHandler) { + UserDataTransferAuditHandler userDataTransferAuditHandler, + UserDataTransferAuditClassifier userDataTransferAuditClassifier) { this.thisNodeEndPoint = thisNode; this.thisNodeId = thisNodeId; this.storageDir = storageDir; @@ -65,6 +67,7 @@ private ConsensusConfig( this.directoryStrategyType = directoryStrategyType; this.trustedChannelFailureHandler = trustedChannelFailureHandler; this.userDataTransferAuditHandler = userDataTransferAuditHandler; + this.userDataTransferAuditClassifier = userDataTransferAuditClassifier; } public TEndPoint getThisNodeEndPoint() { @@ -111,6 +114,10 @@ public UserDataTransferAuditHandler getUserDataTransferAuditHandler() { return userDataTransferAuditHandler; } + public UserDataTransferAuditClassifier getUserDataTransferAuditClassifier() { + return userDataTransferAuditClassifier; + } + public static ConsensusConfig.Builder newBuilder() { return new ConsensusConfig.Builder(); } @@ -131,6 +138,8 @@ public static class Builder { TrustedChannelFailureHandler.NO_OP; private UserDataTransferAuditHandler userDataTransferAuditHandler = UserDataTransferAuditHandler.NO_OP; + private UserDataTransferAuditClassifier userDataTransferAuditClassifier = + UserDataTransferAuditClassifier.NO_USER_DATA; public ConsensusConfig build() { return new ConsensusConfig( @@ -146,7 +155,8 @@ public ConsensusConfig build() { .orElseGet(() -> IoTConsensusV2Config.newBuilder().build()), directoryStrategyType, trustedChannelFailureHandler, - userDataTransferAuditHandler); + userDataTransferAuditHandler, + userDataTransferAuditClassifier); } public Builder setThisNode(TEndPoint thisNode) { @@ -209,5 +219,13 @@ public Builder setUserDataTransferAuditHandler( .orElse(UserDataTransferAuditHandler.NO_OP); return this; } + + public Builder setUserDataTransferAuditClassifier( + UserDataTransferAuditClassifier userDataTransferAuditClassifier) { + this.userDataTransferAuditClassifier = + Optional.ofNullable(userDataTransferAuditClassifier) + .orElse(UserDataTransferAuditClassifier.NO_USER_DATA); + return this; + } } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java new file mode 100644 index 0000000000000..952f0b2125759 --- /dev/null +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.config; + +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.request.IConsensusRequest; + +@FunctionalInterface +public interface UserDataTransferAuditClassifier { + + UserDataTransferAuditClassifier NO_USER_DATA = (groupId, request) -> false; + + boolean containsUserData(ConsensusGroupId groupId, IConsensusRequest request); +} diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java index 0a4f70e4e3da4..ede52c92ed3bc 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java @@ -46,6 +46,7 @@ import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.config.ConsensusConfig; import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.config.UserDataTransferAuditClassifier; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.consensus.exception.ConsensusGroupAlreadyExistException; import org.apache.iotdb.consensus.exception.ConsensusGroupModifyPeerException; @@ -106,6 +107,7 @@ public class IoTConsensus implements IConsensus { private final IoTConsensusRPCService service; private final RegisterManager registerManager = new RegisterManager(); private final UserDataTransferAuditHandler userDataTransferAuditHandler; + private final UserDataTransferAuditClassifier userDataTransferAuditClassifier; private volatile IoTConsensusConfig config; /** @@ -134,6 +136,7 @@ public IoTConsensus(ConsensusConfig config, Registry registry) { this.recvFolderStrategyType = config.getDirectoryStrategyType(); this.config = config.getIotConsensusConfig(); this.userDataTransferAuditHandler = config.getUserDataTransferAuditHandler(); + this.userDataTransferAuditClassifier = config.getUserDataTransferAuditClassifier(); this.registry = registry; this.service = new IoTConsensusRPCService( @@ -211,7 +214,8 @@ private void initAndRecover() throws IOException { clientManager, syncClientManager, config, - userDataTransferAuditHandler); + userDataTransferAuditHandler, + userDataTransferAuditClassifier); stateMachineMap.put(consensusGroupId, consensus); } } catch (DiskSpaceInsufficientException e) { @@ -327,7 +331,8 @@ public void createLocalPeer(ConsensusGroupId groupId, List peers) clientManager, syncClientManager, config, - userDataTransferAuditHandler); + userDataTransferAuditHandler, + userDataTransferAuditClassifier); } catch (DiskSpaceInsufficientException e) { throw new RuntimeException(e); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index f342079878bf9..179e04d95948e 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -24,7 +24,6 @@ import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.consensus.ConsensusGroupId; @@ -46,6 +45,7 @@ import org.apache.iotdb.consensus.common.request.DeserializedBatchIndexedConsensusRequest; import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.config.UserDataTransferAuditClassifier; import org.apache.iotdb.consensus.exception.ConsensusGroupModifyPeerException; import org.apache.iotdb.consensus.i18n.ConsensusMessages; import org.apache.iotdb.consensus.i18n.IoTConsensusMessages; @@ -163,6 +163,7 @@ public class IoTConsensusServerImpl { private final IoTConsensusRateLimiter ioTConsensusRateLimiter = IoTConsensusRateLimiter.getInstance(); private final UserDataTransferAuditHandler userDataTransferAuditHandler; + private final UserDataTransferAuditClassifier userDataTransferAuditClassifier; private IndexedConsensusRequest lastConsensusRequest; // Subscription queues receive IndexedConsensusRequest in real-time from write(), @@ -212,7 +213,8 @@ public IoTConsensusServerImpl( clientManager, syncClientManager, config, - UserDataTransferAuditHandler.NO_OP); + UserDataTransferAuditHandler.NO_OP, + UserDataTransferAuditClassifier.NO_USER_DATA); } public IoTConsensusServerImpl( @@ -226,7 +228,8 @@ public IoTConsensusServerImpl( IClientManager clientManager, IClientManager syncClientManager, IoTConsensusConfig config, - UserDataTransferAuditHandler userDataTransferAuditHandler) + UserDataTransferAuditHandler userDataTransferAuditHandler, + UserDataTransferAuditClassifier userDataTransferAuditClassifier) throws DiskSpaceInsufficientException { this.active = true; this.storageDir = storageDir; @@ -248,6 +251,7 @@ public IoTConsensusServerImpl( this.backgroundTaskService = backgroundTaskService; this.config = config; this.userDataTransferAuditHandler = userDataTransferAuditHandler; + this.userDataTransferAuditClassifier = userDataTransferAuditClassifier; this.consensusGroupId = thisNode.getGroupId().toString(); this.consensusReqReader = (ConsensusReqReader) stateMachine.read(new GetConsensusReqReaderPlan()); @@ -521,23 +525,18 @@ private void recordSnapshotTransferAttempt( boolean success, String errorCode, Throwable error) { - if (!userDataTransferAuditHandler.isEnabled()) { - return; - } try { + if (!userDataTransferAuditHandler.isEnabled()) { + return; + } userDataTransferAuditHandler.onAttempt( new UserDataTransferAuditEvent( - UserDataTransferType.IOT_CONSENSUS_SNAPSHOT, thisNode.getEndpoint(), thisNode.getEndpoint(), targetPeer.getEndpoint(), UserDataTransferProtectionMethod.fromTlsEnabled(config.getRpc().isEnableSSL()), - null, - request.getSnapshotId() + "/" + request.getOffset(), - 1, success, - errorCode, - error)); + errorCode != null ? errorCode : error == null ? null : error.getClass().getName())); } catch (RuntimeException ignored) { // Audit recording must not affect snapshot transmission. } @@ -1014,6 +1013,7 @@ public IndexedConsensusRequest buildIndexedConsensusRequestForLocalRequest( ((ComparableConsensusRequest) request).setProgressIndex(iotProgressIndex); } return new IndexedConsensusRequest(searchIndex.get() + 1, Collections.singletonList(request)) + .setContainsUserData(containsUserData(Collections.singletonList(request))) .setPhysicalTime(assignPhysicalTimeInMs()) .setNodeId(thisNode.getNodeId()); } @@ -1030,9 +1030,23 @@ public IndexedConsensusRequest buildIndexedConsensusRequestForRemoteRequest( req.setRoutingEpoch(routingEpoch); req.setPhysicalTime(physicalTime); req.setNodeId(nodeId); + req.setContainsUserData(containsUserData(requests)); return req; } + public boolean containsUserData(List requests) { + for (IConsensusRequest request : requests) { + try { + if (userDataTransferAuditClassifier.containsUserData(thisNode.getGroupId(), request)) { + return true; + } + } catch (RuntimeException ignored) { + // Classification is advisory and must not affect consensus replication. + } + } + return false; + } + public TSStatus syncIdleWriterSafeTimeBarrierToPeer(final Peer targetPeer) { final long safePhysicalTime = assignPhysicalTimeInMs(); final long safeLocalSeq = searchIndex.get(); diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java index e80be25b4dec5..e85b0241d7af0 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java @@ -19,10 +19,11 @@ package org.apache.iotdb.consensus.iot.client; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; +import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.consensus.i18n.IoTConsensusMessages; import org.apache.iotdb.consensus.iot.logdispatcher.Batch; @@ -187,31 +188,43 @@ private void completeBatch(Batch batch) { } private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { - if (!thread.getImpl().getUserDataTransferAuditHandler().isEnabled()) { - return; + try { + recordTransferAttempt( + thread.getImpl().getUserDataTransferAuditHandler(), + batch, + thread.getImpl().getThisNode().getEndpoint(), + thread.getPeer().getEndpoint(), + UserDataTransferProtectionMethod.fromTlsEnabled( + thread.getConfig().getRpc().isEnableSSL()), + success, + errorCode, + error); + } catch (RuntimeException ignored) { + // Audit recording must not affect consensus replication. } + } + + static void recordTransferAttempt( + UserDataTransferAuditHandler auditHandler, + Batch batch, + TEndPoint source, + TEndPoint target, + UserDataTransferProtectionMethod protectionMethod, + boolean success, + String errorCode, + Throwable error) { try { - thread - .getImpl() - .getUserDataTransferAuditHandler() - .onAttempt( - new UserDataTransferAuditEvent( - UserDataTransferType.IOT_CONSENSUS_LOG, - thread.getImpl().getThisNode().getEndpoint(), - thread.getImpl().getThisNode().getEndpoint(), - thread.getPeer().getEndpoint(), - UserDataTransferProtectionMethod.fromTlsEnabled( - thread.getConfig().getRpc().isEnableSSL()), - null, - thread.getImpl().getThisNode().getGroupId() - + "/" - + batch.getStartIndex() - + "-" - + batch.getEndIndex(), - retryCount + 1, - success, - errorCode, - error)); + if (!batch.containsUserData() || !auditHandler.isEnabled()) { + return; + } + auditHandler.onAttempt( + new UserDataTransferAuditEvent( + source, + source, + target, + protectionMethod, + success, + errorCode != null ? errorCode : error == null ? null : error.getClass().getName())); } catch (RuntimeException ignored) { // Audit recording must not affect consensus replication. } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/Batch.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/Batch.java index 72b68ab96ac7e..55569b8a34fc7 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/Batch.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/Batch.java @@ -39,6 +39,7 @@ public class Batch { private long memorySize; // indicates whether this batch has been successfully synchronized to another node private boolean synced; + private boolean containsUserData; public Batch(IoTConsensusConfig config) { this.config = config; @@ -55,11 +56,16 @@ public void buildIndex() { } public void addTLogEntry(TLogEntry entry) { + addTLogEntry(entry, false); + } + + public void addTLogEntry(TLogEntry entry, boolean containsUserData) { logEntries.add(entry); if (entry.fromWAL) { logEntriesNumFromWAL++; } memorySize += entry.getMemorySize(); + this.containsUserData |= containsUserData; } public boolean canAccumulate() { @@ -107,6 +113,10 @@ public long getLogEntriesNumFromWAL() { return logEntriesNumFromWAL; } + public boolean containsUserData() { + return containsUserData; + } + @Override public String toString() { return "Batch{" diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java index cd9d7eea49f83..4fc9089856e64 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java @@ -675,6 +675,7 @@ private boolean constructBatchFromWAL(long currentIndex, long maxIndex, Batch lo hasCorruptedData = true; } targetIndex = data.getSearchIndex() + 1; + data.setContainsUserData(impl.containsUserData(data.getRequests())); data.buildSerializedRequests(); // construct request from wal TLogEntry logEntry = @@ -682,7 +683,7 @@ private boolean constructBatchFromWAL(long currentIndex, long maxIndex, Batch lo data.getSerializedRequests(), data.getSearchIndex(), true, data.getMemorySize()); logEntry.setRoutingEpoch(data.getRoutingEpoch()); logEntry.setPhysicalTime(data.getPhysicalTime()); - logBatches.addTLogEntry(logEntry); + logBatches.addTLogEntry(logEntry, data.containsUserData()); } // In the case of corrupt Data, we return true so that we can send a batch as soon as // possible, avoiding potential duplication @@ -699,7 +700,7 @@ private void constructBatchIndexedFromConsensusRequest( request.getMemorySize()); logEntry.setRoutingEpoch(request.getRoutingEpoch()); logEntry.setPhysicalTime(request.getPhysicalTime()); - logBatches.addTLogEntry(logEntry); + logBatches.addTLogEntry(logEntry, request.containsUserData()); } } diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java new file mode 100644 index 0000000000000..fa6347abfd731 --- /dev/null +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.iot.client; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; +import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; +import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; +import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.iot.logdispatcher.Batch; +import org.apache.iotdb.consensus.iot.thrift.TLogEntry; + +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DispatchLogHandlerTest { + + private static final TEndPoint SOURCE = new TEndPoint("127.0.0.1", 10740); + private static final TEndPoint TARGET = new TEndPoint("127.0.0.2", 10740); + + @Test + public void testRecordsSuccessRemoteFailureAndException() { + final List events = new ArrayList<>(); + final Batch batch = createBatch(true); + + DispatchLogHandler.recordTransferAttempt( + events::add, batch, SOURCE, TARGET, UserDataTransferProtectionMethod.TLS, true, null, null); + DispatchLogHandler.recordTransferAttempt( + events::add, + batch, + SOURCE, + TARGET, + UserDataTransferProtectionMethod.TLS, + false, + "500", + null); + DispatchLogHandler.recordTransferAttempt( + events::add, + batch, + SOURCE, + TARGET, + UserDataTransferProtectionMethod.TLS, + false, + null, + new IOException()); + + assertEquals(3, events.size()); + assertTrue(events.get(0).isSuccess()); + assertEquals(SOURCE, events.get(0).getInitiator()); + assertEquals(SOURCE, events.get(0).getSource()); + assertEquals(TARGET, events.get(0).getTarget()); + assertEquals(UserDataTransferProtectionMethod.TLS, events.get(0).getProtectionMethod()); + assertFalse(events.get(1).isSuccess()); + assertEquals("500", events.get(1).getError()); + assertEquals(IOException.class.getName(), events.get(2).getError()); + } + + @Test + public void testSkipsBatchWithoutUserData() { + final List events = new ArrayList<>(); + + DispatchLogHandler.recordTransferAttempt( + events::add, + createBatch(false), + SOURCE, + TARGET, + UserDataTransferProtectionMethod.UNPROTECTED, + true, + null, + null); + + assertTrue(events.isEmpty()); + } + + @Test + public void testAuditHandlerFailureDoesNotEscape() { + DispatchLogHandler.recordTransferAttempt( + event -> { + throw new IllegalStateException(); + }, + createBatch(true), + SOURCE, + TARGET, + UserDataTransferProtectionMethod.UNPROTECTED, + true, + null, + null); + + DispatchLogHandler.recordTransferAttempt( + new UserDataTransferAuditHandler() { + @Override + public void onAttempt(UserDataTransferAuditEvent event) { + // Do nothing. + } + + @Override + public boolean isEnabled() { + throw new IllegalStateException(); + } + }, + createBatch(true), + SOURCE, + TARGET, + UserDataTransferProtectionMethod.UNPROTECTED, + true, + null, + null); + } + + private static Batch createBatch(boolean containsUserData) { + final Batch batch = new Batch(IoTConsensusConfig.newBuilder().build()); + batch.addTLogEntry(new TLogEntry().setSearchIndex(1).setMemorySize(1), containsUserData); + batch.buildIndex(); + return batch; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java index cc406bda68a03..d519dba836849 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java @@ -22,9 +22,20 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.request.IConsensusRequest; +import org.apache.iotdb.commons.schema.table.Audit; +import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest; +import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import javax.annotation.Nullable; @@ -39,32 +50,77 @@ public static boolean isEnabled() { } public static void record( - UserDataTransferType transferType, TEndPoint initiator, TEndPoint source, TEndPoint target, - @Nullable String context, - int attempt, boolean success, @Nullable String errorCode, @Nullable Throwable error) { - if (!isEnabled()) { - return; + try { + if (!isEnabled()) { + return; + } + DNAuditLogger.getInstance() + .recordUserDataTransferAuditLog( + new UserDataTransferAuditEvent( + initiator, + source, + target, + UserDataTransferProtectionMethod.fromTlsEnabled( + COMMON_CONFIG.isEnableInternalSSL()), + success, + errorCode != null + ? errorCode + : error == null ? null : error.getClass().getName())); + } catch (RuntimeException ignored) { + // Audit recording must not affect user data transfer. } - DNAuditLogger.getInstance() - .recordUserDataTransferAuditLog( - new UserDataTransferAuditEvent( - transferType, - initiator, - source, - target, - UserDataTransferProtectionMethod.fromTlsEnabled( - COMMON_CONFIG.isEnableInternalSSL()), - COMMON_CONFIG.isEnableInternalSSL() ? COMMON_CONFIG.getSslProtocol() : null, - context, - attempt, - success, - errorCode, - error)); + } + + public static boolean containsUserData( + ConsensusGroupId consensusGroupId, IConsensusRequest request) { + if (!(consensusGroupId instanceof DataRegionId)) { + return false; + } + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion((DataRegionId) consensusGroupId); + return dataRegion != null && containsUserData(dataRegion.getDatabaseName(), request); + } + + static boolean containsUserData(String database, IConsensusRequest request) { + if (Audit.isAuditDatabase(database)) { + return false; + } + try { + final PlanNode planNode; + if (request instanceof PlanNode) { + planNode = (PlanNode) request; + } else if (request instanceof IoTConsensusRequest) { + planNode = WALEntry.deserializeForConsensus(request.serializeToByteBuffer().duplicate()); + } else if (request instanceof ByteBufferConsensusRequest) { + planNode = PlanNodeType.deserialize(request.serializeToByteBuffer().duplicate()); + } else { + return false; + } + return containsInsertNode(planNode); + } catch (RuntimeException ignored) { + // Classification is advisory and must not affect consensus replication. + return false; + } + } + + public static boolean containsInsertNode(PlanNode node) { + if (node instanceof InsertNode) { + return true; + } + if (node.getChildren() == null) { + return false; + } + for (PlanNode child : node.getChildren()) { + if (containsInsertNode(child)) { + return true; + } + } + return false; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index 0a16f643b6ace..14bfb078fd96b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -41,6 +41,7 @@ import org.apache.iotdb.consensus.config.RatisConfig; import org.apache.iotdb.consensus.config.RatisConfig.Snapshot; import org.apache.iotdb.db.audit.DNAuditLogger; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -148,6 +149,7 @@ private static ConsensusConfig buildConsensusConfig() { COMMON_CONF.isEnableAuditLog() ? DNAuditLogger.getInstance()::recordUserDataTransferAuditLog : UserDataTransferAuditHandler.NO_OP) + .setUserDataTransferAuditClassifier(DataNodeUserDataTransferAuditor::containsUserData) .setStorageDir(CONF.getDataRegionConsensusDir()) .setRecvSnapshotDirs(Arrays.asList(CONF.getLocalDataDirs())) // IoTConsensus always balances received snapshot files by least occupied space, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java index 4faf31f3b702f..99bd0ed401796 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java @@ -22,7 +22,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.consensus.ConsensusGroupId; @@ -748,29 +747,13 @@ public TEndPoint getFollowerUrl() { return nodeUrls.get(0); } - public void recordUserDataTransferAudit( - UserDataTransferType transferType, - String context, - boolean success, - String errorCode, - Throwable error) { - if (!DataNodeUserDataTransferAuditor.isEnabled()) { - return; - } + public void recordUserDataTransferAudit(boolean success, String errorCode, Throwable error) { final TEndPoint localEndPoint = new TEndPoint( IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusPort()); DataNodeUserDataTransferAuditor.record( - transferType, - localEndPoint, - localEndPoint, - getFollowerUrl(), - context, - 1, - success, - errorCode, - error); + localEndPoint, localEndPoint, getFollowerUrl(), success, errorCode, error); } // synchronized to avoid close connector when transfer event diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java index 3641e6c9c1634..a8a8dcc8d126e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java @@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.consensus.index.ProgressIndex; @@ -217,8 +216,6 @@ private void doTransfer() { final TSStatus failedStatus = statusList.stream().filter(status -> !isSuccessful(status)).findFirst().orElse(null); recordTransferAttempt( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - String.valueOf(consensusGroupId), failedStatus == null, failedStatus == null ? null : String.valueOf(failedStatus.getCode()), null); @@ -239,12 +236,7 @@ private void doTransfer() { tabletBatchBuilder.onSuccess(); } catch (final Exception e) { if (!transferAttemptRecorded) { - recordTransferAttempt( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - String.valueOf(consensusGroupId), - false, - null, - e); + recordTransferAttempt(false, null, e); } throw new PipeRuntimeSinkRetryTimesConfigurableException( String.format( @@ -366,20 +358,13 @@ private void doTransfer(PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletI resp = syncIoTConsensusV2ServiceClient.iotConsensusV2Transfer(req); final TSStatus status = resp.getStatus(); recordTransferAttempt( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - String.valueOf(pipeInsertNodeTabletInsertionEvent.getReplicateIndexForIoTV2()), isSuccessful(status), isSuccessful(status) ? null : String.valueOf(status.getCode()), null); transferAttemptRecorded = true; } catch (final Exception e) { if (!transferAttemptRecorded) { - recordTransferAttempt( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - String.valueOf(pipeInsertNodeTabletInsertionEvent.getReplicateIndexForIoTV2()), - false, - null, - e); + recordTransferAttempt(false, null, e); } throw new PipeRuntimeSinkRetryTimesConfigurableException( String.format( @@ -528,20 +513,13 @@ protected void transferFilePieces( thisDataNodeId))); final TSStatus transferStatus = resp.getStatus(); recordTransferAttempt( - UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, - file.getName() + "/" + transferPosition, isSuccessful(transferStatus), isSuccessful(transferStatus) ? null : String.valueOf(transferStatus.getCode()), null); transferAttemptRecorded = true; } catch (Exception e) { if (!transferAttemptRecorded) { - recordTransferAttempt( - UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, - file.getName() + "/" + transferPosition, - false, - null, - e); + recordTransferAttempt(false, null, e); } throw new PipeRuntimeSinkRetryTimesConfigurableException( String.format( @@ -596,29 +574,13 @@ private static boolean isSuccessful(TSStatus status) { || status.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(); } - private void recordTransferAttempt( - UserDataTransferType transferType, - String context, - boolean success, - String errorCode, - Throwable error) { - if (!DataNodeUserDataTransferAuditor.isEnabled()) { - return; - } + private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { final TEndPoint localEndPoint = new TEndPoint( IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusPort()); DataNodeUserDataTransferAuditor.record( - transferType, - localEndPoint, - localEndPoint, - getFollowerUrl(), - context, - 1, - success, - errorCode, - error); + localEndPoint, localEndPoint, getFollowerUrl(), success, errorCode, error); } // synchronized to avoid close connector when transfer event diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java index 8b1cf0e19a8d7..2ccefdeb2ed9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.handler; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -92,10 +91,6 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { .findFirst() .orElse(null); connector.recordUserDataTransferAudit( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - requestCommitIds.isEmpty() - ? null - : requestCommitIds.get(0) + "/" + requestCommitIds.size(), failedStatus == null, failedStatus == null ? null : String.valueOf(failedStatus.getCode()), null); @@ -135,14 +130,7 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { @Override public void onError(final Exception exception) { if (!transferAuditRecorded) { - connector.recordUserDataTransferAudit( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - requestCommitIds.isEmpty() - ? null - : requestCommitIds.get(0) + "/" + requestCommitIds.size(), - false, - null, - exception); + connector.recordUserDataTransferAudit(false, null, exception); transferAuditRecorded = true; } final Object pipeNames = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java index 347435fe366de..794414c9d30fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletInsertionEventHandler.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.handler; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -89,11 +88,7 @@ public void onComplete(TIoTConsensusV2TransferResp response) { status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() || status.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(); connector.recordUserDataTransferAudit( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - String.valueOf(((EnrichedEvent) event).getReplicateIndexForIoTV2()), - success, - success ? null : String.valueOf(status.getCode()), - null); + success, success ? null : String.valueOf(status.getCode()), null); transferAuditRecorded = true; try { // Only handle the failed statuses to avoid string format performance overhead @@ -126,12 +121,7 @@ public void onComplete(TIoTConsensusV2TransferResp response) { @Override public void onError(Exception exception) { if (!transferAuditRecorded) { - connector.recordUserDataTransferAudit( - UserDataTransferType.IOT_CONSENSUS_V2_TABLET, - String.valueOf(((EnrichedEvent) event).getReplicateIndexForIoTV2()), - false, - null, - exception); + connector.recordUserDataTransferAudit(false, null, exception); transferAuditRecorded = true; } EnrichedEvent event = (EnrichedEvent) this.event; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java index 9e561b6617f23..f350ff8c78f69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TsFileInsertionEventHandler.java @@ -21,7 +21,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -90,7 +89,6 @@ public class IoTConsensusV2TsFileInsertionEventHandler private long startTransferPieceTime; private boolean currentAttemptContainsUserData; private boolean transferAuditRecorded; - private String transferAuditContext; public IoTConsensusV2TsFileInsertionEventHandler( final PipeTsFileInsertionEvent event, @@ -198,7 +196,6 @@ public void transfer(final AsyncIoTConsensusV2ServiceClient client) : Arrays.copyOfRange(readBuffer, 0, readLength); currentAttemptContainsUserData = true; transferAuditRecorded = false; - transferAuditContext = currentFile.getName() + "/" + position; client.iotConsensusV2Transfer( transferMod ? IoTConsensusV2TsFilePieceWithModReq.toTIoTConsensusV2TransferReq( @@ -288,11 +285,7 @@ public void onComplete(final TIoTConsensusV2TransferResp response) { transferStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() || transferStatus.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(); connector.recordUserDataTransferAudit( - UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, - transferAuditContext, - success, - success ? null : String.valueOf(transferStatus.getCode()), - null); + success, success ? null : String.valueOf(transferStatus.getCode()), null); transferAuditRecorded = true; // This case only happens when the connection is broken, and the connector is reconnected @@ -329,12 +322,7 @@ public void onComplete(final TIoTConsensusV2TransferResp response) { @Override public void onError(final Exception exception) { if (currentAttemptContainsUserData && !transferAuditRecorded) { - connector.recordUserDataTransferAudit( - UserDataTransferType.IOT_CONSENSUS_V2_TSFILE, - transferAuditContext, - false, - null, - exception); + connector.recordUserDataTransferAudit(false, null, exception); transferAuditRecorded = true; } PipeLogger.log( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java index d3ca71b89ad37..53f1fff225201 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java @@ -21,7 +21,6 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.audit.UserDataTransferErrorCode; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; import org.apache.iotdb.commons.utils.TestOnly; @@ -650,9 +649,13 @@ public void run() { mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint)) { TGetDataBlockResponse resp = client.getDataBlock(req); int tsBlockNum = resp.getTsBlocks().size(); - if (tsBlockNum == 0) { + if (tsBlockNum != endSequenceId - startSequenceId) { recordTransferAttempt( - attempt, false, UserDataTransferErrorCode.EMPTY_RESPONSE.name(), null); + false, + tsBlockNum == 0 + ? UserDataTransferErrorCode.EMPTY_RESPONSE.name() + : UserDataTransferErrorCode.UNEXPECTED_RESPONSE_SIZE.name(), + null); transferAttemptRecorded = true; if (!closed) { // failed to pull TsBlocks @@ -666,8 +669,6 @@ public void run() { } return; } - recordTransferAttempt(attempt, true, null, null); - transferAttemptRecorded = true; List tsBlocks = new ArrayList<>(tsBlockNum); tsBlocks.addAll(resp.getTsBlocks()); @@ -678,25 +679,35 @@ public void run() { GET_DATA_BLOCK_NUM_CALLER, tsBlockNum); executorService.submit( new SendAcknowledgeDataBlockEventTask(startSequenceId, endSequenceId)); + boolean receiverClosed = false; synchronized (SourceHandle.this) { if (aborted || closed) { - return; - } - for (int i = startSequenceId; i < endSequenceId; i++) { - sequenceIdToTsBlock.put(i, tsBlocks.get(i - startSequenceId)); - } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(DataNodeQueryMessages.PUT_TSBLOCKS_INTO_BUFFER); - } - if (!blocked.isDone()) { - blocked.set(null); + receiverClosed = true; + } else { + for (int i = startSequenceId; i < endSequenceId; i++) { + sequenceIdToTsBlock.put(i, tsBlocks.get(i - startSequenceId)); + } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(DataNodeQueryMessages.PUT_TSBLOCKS_INTO_BUFFER); + } + if (!blocked.isDone()) { + blocked.set(null); + } } } + recordTransferAttempt( + !receiverClosed, + receiverClosed ? UserDataTransferErrorCode.RECEIVER_CLOSED.name() : null, + null); + transferAttemptRecorded = true; + if (receiverClosed) { + return; + } break; } catch (Throwable e) { if (!transferAttemptRecorded) { - recordTransferAttempt(attempt, false, null, e); + recordTransferAttempt(false, null, e); } LOGGER.warn( @@ -728,27 +739,9 @@ public void run() { } } - private void recordTransferAttempt( - int attempt, boolean success, String errorCode, Throwable error) { - if (!DataNodeUserDataTransferAuditor.isEnabled()) { - return; - } + private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { DataNodeUserDataTransferAuditor.record( - UserDataTransferType.MPP_TS_BLOCK, - localEndpoint, - remoteEndpoint, - localEndpoint, - remoteFragmentInstanceId - + "/" - + indexOfUpstreamSinkHandle - + "/" - + startSequenceId - + "-" - + endSequenceId, - attempt, - success, - errorCode, - error); + localEndpoint, remoteEndpoint, localEndpoint, success, errorCode, error); } private void fail(Throwable t) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java index 31bdae0728fea..2346cfd637d9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java @@ -21,13 +21,14 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.mpp.rpc.thrift.TPlanNode; import org.apache.iotdb.mpp.rpc.thrift.TSendBatchPlanNodeReq; import org.apache.iotdb.mpp.rpc.thrift.TSendSinglePlanNodeReq; @@ -61,7 +62,6 @@ public class AsyncPlanNodeSender { private final AtomicLong pendingNumber; private long startSendTime; - private int auditAttempt; public AsyncPlanNodeSender( IClientManager @@ -86,7 +86,7 @@ public AsyncPlanNodeSender( new TPlanNode( instances.get(i).getFragment().getPlanNodeTree().serializeToByteBuffer()), instances.get(i).getRegionReplicaSet().getRegionId()), - containsInsertNode(instances.get(i).getFragment().getPlanNodeTree())); + containsUserData(instances.get(i))); } this.instanceId2RespMap = new ConcurrentHashMap<>(instances.size() + 1, 1); this.needRetryInstanceIndex = Collections.synchronizedList(new ArrayList<>()); @@ -94,7 +94,6 @@ public AsyncPlanNodeSender( } public void sendAll() { - auditAttempt++; for (Map.Entry entry : batchRequests.entrySet()) { AsyncSendPlanNodeHandler handler = new AsyncSendPlanNodeHandler( @@ -105,9 +104,7 @@ public void sendAll() { startSendTime, localEndPoint, entry.getKey(), - entry.getValue().containsUserData(), - buildAuditContext(entry.getValue()), - auditAttempt); + entry.getValue().containsUserData()); try { AsyncDataNodeInternalServiceClient client = asyncInternalServiceClientManager.borrowClient(entry.getKey()); @@ -205,8 +202,7 @@ public void retry() throws InterruptedException { .getPlanNodeTree() .serializeToByteBuffer()), instances.get(fragmentInstanceIndex).getRegionReplicaSet().getRegionId()), - containsInsertNode( - instances.get(fragmentInstanceIndex).getFragment().getPlanNodeTree())); + containsUserData(instances.get(fragmentInstanceIndex))); } // 2. reset the pendingNumber, needRetryInstanceIds and startSendTime @@ -253,27 +249,18 @@ public boolean containsUserData() { } } - private String buildAuditContext(BatchRequestWithIndex batchRequest) { - if (batchRequest.getIndexes().isEmpty()) { - return null; - } - return instances.get(batchRequest.getIndexes().get(0)).getId().getFullId() - + "/" - + batchRequest.getIndexes().size(); + private static boolean containsUserData(FragmentInstance instance) { + return containsUserData( + instance.getFragment().getPlanNodeTree(), + instance.getSessionInfo() == null ? null : instance.getSessionInfo().getUserName()); + } + + static boolean containsUserData(PlanNode node, String username) { + return !User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME.equals(username) + && DataNodeUserDataTransferAuditor.containsInsertNode(node); } static boolean containsInsertNode(PlanNode node) { - if (node instanceof InsertNode) { - return true; - } - if (node.getChildren() == null) { - return false; - } - for (PlanNode child : node.getChildren()) { - if (containsInsertNode(child)) { - return true; - } - } - return false; + return DataNodeUserDataTransferAuditor.containsInsertNode(node); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java index fe9ce662a3fd2..eae8dbc00910e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncSendPlanNodeHandler.java @@ -21,7 +21,6 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.audit.UserDataTransferErrorCode; -import org.apache.iotdb.commons.audit.UserDataTransferType; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; @@ -50,8 +49,6 @@ public class AsyncSendPlanNodeHandler implements AsyncMethodCallback dispatchCommand( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java new file mode 100644 index 0000000000000..9268f57b0c375 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.audit; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; + +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DataNodeUserDataTransferAuditorTest { + + @Test + public void testAuditDatabaseIsExcludedFromConsensusTransferAudit() { + final InsertNode insertNode = mock(InsertNode.class); + + assertFalse(DataNodeUserDataTransferAuditor.containsUserData("__audit", insertNode)); + assertFalse(DataNodeUserDataTransferAuditor.containsUserData("root.__audit", insertNode)); + assertTrue(DataNodeUserDataTransferAuditor.containsUserData("root.sg", insertNode)); + } + + @Test + public void testNonInsertConsensusRequestIsExcluded() { + final PlanNode planNode = mock(PlanNode.class); + when(planNode.getChildren()).thenReturn(Collections.emptyList()); + + assertFalse(DataNodeUserDataTransferAuditor.containsUserData("root.sg", planNode)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java index ff4ddb27354fa..4ef63bca9d8a3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.scheduler; +import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; @@ -41,6 +42,9 @@ public void testOnlyInsertPayloadIsClassifiedAsUserData() { final InsertNode insertNode = mock(InsertNode.class); assertTrue(AsyncPlanNodeSender.containsInsertNode(insertNode)); + assertTrue(AsyncPlanNodeSender.containsUserData(insertNode, "root")); + assertFalse( + AsyncPlanNodeSender.containsUserData(insertNode, User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME)); final PlanNode wrapper = mock(PlanNode.class); when(wrapper.getChildren()).thenReturn(Collections.singletonList(insertNode)); diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index 9181235b1cb33..fb750ba4cfc66 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -225,9 +225,8 @@ private CommonMessages() {} public static final String LOG_TRUSTED_CHANNEL_FUNCTION_FAILED_INITIATOR_ARG_TARGET_ARG_E4C28443 = "Trusted channel function failed: initiator=%s, target=%s"; public static final String - LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_TYPE_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_PROTECTION_PROTOCOL_ARG_CONTEXT_ARG_ATTEMPT_ARG_ERROR_CODE_ARG_ERROR_TYPE_ARG_941238A8 = - "User data transfer attempt: time=%d, type=%s, initiator=%s, source=%s, target=%s," - + " protection_method=%s, protection_protocol=%s, context=%s, attempt=%d," - + " error_code=%s, error_type=%s"; + LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_RESULT_ARG_ERROR_ARG_D3E9A1DF = + "User data transfer attempt: time=%d, initiator=%s, source=%s, target=%s," + + " protection_method=%s, result=%s, error=%s"; } diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index 0353c395700a0..9a1bc8029e343 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -218,8 +218,7 @@ private CommonMessages() {} public static final String LOG_TRUSTED_CHANNEL_FUNCTION_FAILED_INITIATOR_ARG_TARGET_ARG_E4C28443 = "可信信道功能失效：发起者=%s，目标端=%s"; public static final String - LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_TYPE_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_PROTECTION_PROTOCOL_ARG_CONTEXT_ARG_ATTEMPT_ARG_ERROR_CODE_ARG_ERROR_TYPE_ARG_941238A8 = - "用户数据传送尝试：时间=%d，类型=%s，发起者=%s，源端=%s，目标端=%s，保护方法=%s，保护协议=%s，" - + "上下文=%s，尝试次数=%d，错误码=%s，错误类型=%s"; + LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_RESULT_ARG_ERROR_ARG_D3E9A1DF = + "用户数据传送尝试：时间=%d，发起者=%s，源端=%s，目标端=%s，保护方法=%s，结果=%s，错误=%s"; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java index 7af0c31cfe159..b0ad1839da1a6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java @@ -161,26 +161,19 @@ public void recordUserDataTransferAuditLog(UserDataTransferAuditEvent event) { User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME, initiatorIdentifier, AuditEventType.USER_DATA_TRANSFER, - event.getTransferType().getOperation(), - event.getTransferType().getPrivilegeType(), - event.isSuccess(), null, - null), + event.isSuccess()), () -> String.format( CommonMessages - .LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_TYPE_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_PROTECTION_PROTOCOL_ARG_CONTEXT_ARG_ATTEMPT_ARG_ERROR_CODE_ARG_ERROR_TYPE_ARG_941238A8, + .LOG_USER_DATA_TRANSFER_ATTEMPT_TIME_ARG_INITIATOR_ARG_SOURCE_ARG_TARGET_ARG_PROTECTION_METHOD_ARG_RESULT_ARG_ERROR_ARG_D3E9A1DF, event.getTimestamp(), - event.getTransferType(), initiatorIdentifier, sourceIdentifier, targetIdentifier, event.getProtectionMethod(), - event.getProtectionProtocol(), - event.getContext(), - event.getAttempt(), - event.getErrorCode(), - event.getErrorType())); + event.isSuccess(), + event.getError())); } catch (RuntimeException ignored) { // Audit recording must not affect the user-data transfer being audited. } finally { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java index f934bec22b988..7564ff044d9a6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEvent.java @@ -30,52 +30,33 @@ public final class UserDataTransferAuditEvent { private final long timestamp; - private final UserDataTransferType transferType; private final TEndPoint initiator; private final TEndPoint source; private final TEndPoint target; private final UserDataTransferProtectionMethod protectionMethod; - private final String protectionProtocol; - private final String context; - private final int attempt; private final boolean success; - private final String errorCode; - private final String errorType; + private final String error; public UserDataTransferAuditEvent( - UserDataTransferType transferType, TEndPoint initiator, TEndPoint source, TEndPoint target, UserDataTransferProtectionMethod protectionMethod, - @Nullable String protectionProtocol, - @Nullable String context, - int attempt, boolean success, - @Nullable String errorCode, - @Nullable Throwable error) { + @Nullable String error) { this.timestamp = System.currentTimeMillis(); - this.transferType = transferType; this.initiator = initiator; this.source = source; this.target = target; this.protectionMethod = protectionMethod; - this.protectionProtocol = protectionProtocol; - this.context = context; - this.attempt = attempt; this.success = success; - this.errorCode = errorCode; - this.errorType = error == null ? null : error.getClass().getName(); + this.error = error; } public long getTimestamp() { return timestamp; } - public UserDataTransferType getTransferType() { - return transferType; - } - public TEndPoint getInitiator() { return initiator; } @@ -92,31 +73,12 @@ public UserDataTransferProtectionMethod getProtectionMethod() { return protectionMethod; } - @Nullable - public String getProtectionProtocol() { - return protectionProtocol; - } - - @Nullable - public String getContext() { - return context; - } - - public int getAttempt() { - return attempt; - } - public boolean isSuccess() { return success; } @Nullable - public String getErrorCode() { - return errorCode; - } - - @Nullable - public String getErrorType() { - return errorType; + public String getError() { + return error; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java index a5e0619b741a3..171a90cee2081 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferAuditHandler.java @@ -35,8 +35,13 @@ public boolean isEnabled() { } }; + /** + * Records one transfer attempt. Implementations must return promptly and must not throw because + * callers invoke this method on data-transfer paths. + */ void onAttempt(UserDataTransferAuditEvent event); + /** Returns whether transfer audit is enabled. Implementations must not throw. */ default boolean isEnabled() { return true; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java index 70e461887df42..2bf187999edad 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferErrorCode.java @@ -21,5 +21,7 @@ public enum UserDataTransferErrorCode { EMPTY_RESPONSE, + UNEXPECTED_RESPONSE_SIZE, + RECEIVER_CLOSED, REMOTE_REJECTED } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java deleted file mode 100644 index 9219b6a71b5b6..0000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.audit; - -import org.apache.iotdb.commons.auth.entity.PrivilegeType; - -public enum UserDataTransferType { - MPP_TS_BLOCK(AuditLogOperation.QUERY, PrivilegeType.READ_DATA), - INSERT_PLAN_NODE(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), - LOAD_TSFILE_PIECE(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), - IOT_CONSENSUS_LOG(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), - IOT_CONSENSUS_SNAPSHOT(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), - IOT_CONSENSUS_V2_TABLET(AuditLogOperation.DML, PrivilegeType.WRITE_DATA), - IOT_CONSENSUS_V2_TSFILE(AuditLogOperation.DML, PrivilegeType.WRITE_DATA); - - private final AuditLogOperation operation; - private final PrivilegeType privilegeType; - - UserDataTransferType(AuditLogOperation operation, PrivilegeType privilegeType) { - this.operation = operation; - this.privilegeType = privilegeType; - } - - public AuditLogOperation getOperation() { - return operation; - } - - public PrivilegeType getPrivilegeType() { - return privilegeType; - } -} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java index 47c392ea9cadc..676d778472263 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java @@ -27,29 +27,21 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; public class UserDataTransferAuditEventTest { @Test - public void testRecordsErrorTypeWithoutErrorMessage() { + public void testRecordsOnlyMinimumTransferFields() { final UserDataTransferAuditEvent event = new UserDataTransferAuditEvent( - UserDataTransferType.MPP_TS_BLOCK, new TEndPoint("127.0.0.1", 10740), new TEndPoint("127.0.0.2", 10740), new TEndPoint("127.0.0.1", 10740), UserDataTransferProtectionMethod.TLS, - "TLSv1.3", - "query/0-1", - 2, false, - null, - new IOException("payload must not be retained")); + IOException.class.getName()); - assertEquals(IOException.class.getName(), event.getErrorType()); - assertNull(event.getErrorCode()); - assertEquals(2, event.getAttempt()); + assertEquals(IOException.class.getName(), event.getError()); assertFalse(event.isSuccess()); } } From e9aa6bd771203066c1846b2f15f1b0eb75e4998c Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 2 Sep 2026 17:10:40 +0800 Subject: [PATCH 3/8] [Feature] Address second-pass audit review --- .../iot/client/DispatchLogHandlerTest.java | 6 +- .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../db/consensus/DataRegionConsensusImpl.java | 6 +- .../exchange/source/SourceHandle.java | 7 +- .../execution/exchange/SourceHandleTest.java | 74 +++++++++++++++++++ .../commons/audit/AbstractAuditLogger.java | 19 +++-- .../UserDataTransferProtectionMethod.java | 4 +- .../audit/AbstractAuditLoggerTest.java | 19 +++++ .../audit/UserDataTransferAuditEventTest.java | 10 +++ 10 files changed, 135 insertions(+), 14 deletions(-) diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java index fa6347abfd731..3d39c89bd7e03 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java @@ -88,7 +88,7 @@ public void testSkipsBatchWithoutUserData() { createBatch(false), SOURCE, TARGET, - UserDataTransferProtectionMethod.UNPROTECTED, + UserDataTransferProtectionMethod.NONE, true, null, null); @@ -105,7 +105,7 @@ public void testAuditHandlerFailureDoesNotEscape() { createBatch(true), SOURCE, TARGET, - UserDataTransferProtectionMethod.UNPROTECTED, + UserDataTransferProtectionMethod.NONE, true, null, null); @@ -125,7 +125,7 @@ public boolean isEnabled() { createBatch(true), SOURCE, TARGET, - UserDataTransferProtectionMethod.UNPROTECTED, + UserDataTransferProtectionMethod.NONE, true, null, null); diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 3feefd72a6f46..3d129ba8f1786 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -1398,6 +1398,8 @@ public final class DataNodeQueryMessages { "{} failed to pull TsBlocks [{}] to [{}] from SinkHandle {}, channel index {},"; public static final String FAILED_TO_GET_DATA_BLOCK = "failed to get data block [{}, {}), attempt times: {}"; + public static final String EXCEPTION_UNEXPECTED_DATA_BLOCK_RESPONSE_SIZE_A7DD7E33 = + "Unexpected data block response size."; public static final String FAILED_TO_SEND_ACK_DATA_BLOCK_EVENT = "failed to send ack data block event [{}, {}), attempt times: {}"; public static final String SEND_CLOSE_SINK_CHANNEL_EVENT_FAILED = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 0d4e7cc0071bf..88b7b0edb9028 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -1379,6 +1379,8 @@ public final class DataNodeQueryMessages { "{} 从 SinkHandle {} 的通道索引 {} 拉取 TsBlocks [{}] 到 [{}] 失败，"; public static final String FAILED_TO_GET_DATA_BLOCK = "获取数据块 [{}, {}) 失败，尝试次数：{}"; + public static final String EXCEPTION_UNEXPECTED_DATA_BLOCK_RESPONSE_SIZE_A7DD7E33 = + "数据块响应数量异常。"; public static final String FAILED_TO_SEND_ACK_DATA_BLOCK_EVENT = "发送数据块确认事件 [{}, {}) 失败，尝试次数：{}"; public static final String SEND_CLOSE_SINK_CHANNEL_EVENT_FAILED = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index 14bfb078fd96b..218e0fbcc1c34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -40,6 +40,7 @@ import org.apache.iotdb.consensus.config.IoTConsensusV2Config.ReplicateMode; import org.apache.iotdb.consensus.config.RatisConfig; import org.apache.iotdb.consensus.config.RatisConfig.Snapshot; +import org.apache.iotdb.consensus.config.UserDataTransferAuditClassifier; import org.apache.iotdb.db.audit.DNAuditLogger; import org.apache.iotdb.db.audit.DataNodeUserDataTransferAuditor; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; @@ -149,7 +150,10 @@ private static ConsensusConfig buildConsensusConfig() { COMMON_CONF.isEnableAuditLog() ? DNAuditLogger.getInstance()::recordUserDataTransferAuditLog : UserDataTransferAuditHandler.NO_OP) - .setUserDataTransferAuditClassifier(DataNodeUserDataTransferAuditor::containsUserData) + .setUserDataTransferAuditClassifier( + COMMON_CONF.isEnableAuditLog() + ? DataNodeUserDataTransferAuditor::containsUserData + : UserDataTransferAuditClassifier.NO_USER_DATA) .setStorageDir(CONF.getDataRegionConsensusDir()) .setRecvSnapshotDirs(Arrays.asList(CONF.getLocalDataDirs())) // IoTConsensus always balances received snapshot files by least occupied space, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java index 53f1fff225201..67a5defb09ac3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java @@ -42,6 +42,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import org.apache.thrift.TException; import org.apache.tsfile.external.commons.lang3.Validate; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TsBlockSerde; @@ -667,7 +668,11 @@ public void run() { remoteFragmentInstanceId, indexOfUpstreamSinkHandle); } - return; + if (tsBlockNum == 0) { + return; + } + throw new TException( + DataNodeQueryMessages.EXCEPTION_UNEXPECTED_DATA_BLOCK_RESPONSE_SIZE_A7DD7E33); } List tsBlocks = new ArrayList<>(tsBlockNum); tsBlocks.addAll(resp.getTsBlocks()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java index 66d50675ddff1..03220a3d10bf9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java @@ -610,6 +610,80 @@ public void testFailedReceive() { .onAborted(sourceHandle); } + @Test + public void testShortResponseRetriesAndFails() { + final String queryId = "q0"; + final int numOfMockTsBlock = 10; + final TEndPoint remoteEndpoint = + new TEndPoint("remote", IoTDBDescriptor.getInstance().getConfig().getMppDataExchangePort()); + final TFragmentInstanceId remoteFragmentInstanceId = new TFragmentInstanceId(queryId, 1, "0"); + final String localPlanNodeId = "exchange_0"; + final TFragmentInstanceId localFragmentInstanceId = new TFragmentInstanceId(queryId, 0, "0"); + + final LocalMemoryManager mockLocalMemoryManager = Mockito.mock(LocalMemoryManager.class); + final MemoryPool mockMemoryPool = Utils.createMockNonBlockedMemoryPool(); + Mockito.when(mockLocalMemoryManager.getQueryPool()).thenReturn(mockMemoryPool); + final SourceHandleListener mockSourceHandleListener = Mockito.mock(SourceHandleListener.class); + final TsBlockSerde mockTsBlockSerde = Utils.createMockTsBlockSerde(MOCK_TSBLOCK_SIZE); + final IClientManager mockClientManager = + Mockito.mock(IClientManager.class); + final SyncDataNodeMPPDataExchangeServiceClient mockClient = + Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); + try { + Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient); + Mockito.doAnswer( + invocation -> { + final TGetDataBlockRequest request = invocation.getArgument(0); + final List shortResponse = new ArrayList<>(); + for (int i = 0; + i < request.getEndSequenceId() - request.getStartSequenceId() - 1; + i++) { + shortResponse.add(ByteBuffer.allocate(0)); + } + return new TGetDataBlockResponse(shortResponse); + }) + .when(mockClient) + .getDataBlock(Mockito.any(TGetDataBlockRequest.class)); + } catch (ClientManagerException | TException e) { + Assert.fail(e.getMessage()); + } + + final SourceHandle sourceHandle = + new SourceHandle( + remoteEndpoint, + remoteFragmentInstanceId, + localFragmentInstanceId, + localPlanNodeId, + 0, + mockLocalMemoryManager, + Executors.newSingleThreadExecutor(), + mockTsBlockSerde, + mockSourceHandleListener, + mockClientManager); + sourceHandle.setRetryIntervalInMs(0L); + final Future blocked = sourceHandle.isBlocked(); + + sourceHandle.updatePendingDataBlockInfo( + 0, + Stream.generate(() -> MOCK_TSBLOCK_SIZE) + .limit(numOfMockTsBlock) + .collect(Collectors.toList())); + + try { + Mockito.verify(mockClient, Mockito.timeout(10_000).times(SourceHandle.MAX_ATTEMPT_TIMES)) + .getDataBlock(Mockito.any(TGetDataBlockRequest.class)); + } catch (TException e) { + Assert.fail(e.getMessage()); + } + Mockito.verify(mockSourceHandleListener, Mockito.timeout(10_000).times(1)) + .onFailure(Mockito.eq(sourceHandle), Mockito.any(TException.class)); + Assert.assertFalse(blocked.isDone()); + Assert.assertEquals(0L, sourceHandle.getBufferRetainedSizeInBytes()); + + sourceHandle.abort(); + Assert.assertTrue(blocked.isDone()); + } + @Test public void testForceClose() { final String queryId = "q0"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java index b0ad1839da1a6..48d0e2811dba1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/AbstractAuditLogger.java @@ -156,13 +156,7 @@ public void recordUserDataTransferAuditLog(UserDataTransferAuditEvent event) { RECORDING_USER_DATA_TRANSFER.set(true); try { log( - new AuditLogFields( - INTERNAL_AUDIT_LOG_USER_ID, - User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME, - initiatorIdentifier, - AuditEventType.USER_DATA_TRANSFER, - null, - event.isSuccess()), + createUserDataTransferAuditLogFields(event, initiatorIdentifier), () -> String.format( CommonMessages @@ -180,4 +174,15 @@ public void recordUserDataTransferAuditLog(UserDataTransferAuditEvent event) { RECORDING_USER_DATA_TRANSFER.remove(); } } + + static AuditLogFields createUserDataTransferAuditLogFields( + UserDataTransferAuditEvent event, String initiatorIdentifier) { + return new AuditLogFields( + INTERNAL_AUDIT_LOG_USER_ID, + User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME, + initiatorIdentifier, + AuditEventType.USER_DATA_TRANSFER, + AuditLogOperation.CONTROL, + event.isSuccess()); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java index 304e3d8d221f3..9f87afa55b4bc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/audit/UserDataTransferProtectionMethod.java @@ -21,9 +21,9 @@ public enum UserDataTransferProtectionMethod { TLS, - UNPROTECTED; + NONE; public static UserDataTransferProtectionMethod fromTlsEnabled(boolean tlsEnabled) { - return tlsEnabled ? TLS : UNPROTECTED; + return tlsEnabled ? TLS : NONE; } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/AbstractAuditLoggerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/AbstractAuditLoggerTest.java index fcb3dbf6d1c06..75de1513a9fa6 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/AbstractAuditLoggerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/AbstractAuditLoggerTest.java @@ -122,6 +122,25 @@ public void testAuditFailureShouldNotReplaceTrustedChannelFailure() { assertSame(auditFailure, channelFailure.getSuppressed()[0]); } + @Test + public void testUserDataTransferUsesControlOperation() { + final UserDataTransferAuditEvent event = + new UserDataTransferAuditEvent( + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.2", 10740), + new TEndPoint("127.0.0.1", 10740), + UserDataTransferProtectionMethod.NONE, + true, + null); + + final AuditLogFields auditLogFields = + AbstractAuditLogger.createUserDataTransferAuditLogFields(event, "127.0.0.1:10740"); + + assertEquals(AuditEventType.USER_DATA_TRANSFER, auditLogFields.getAuditEventType()); + assertEquals(AuditLogOperation.CONTROL, auditLogFields.getAuditLogOperation()); + assertTrue(auditLogFields.getResult()); + } + private static class TestAuditLogger extends AbstractAuditLogger { private IAuditEntity auditEntity; diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java index 676d778472263..423b05cf9d4ae 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/audit/UserDataTransferAuditEventTest.java @@ -44,4 +44,14 @@ public void testRecordsOnlyMinimumTransferFields() { assertEquals(IOException.class.getName(), event.getError()); assertFalse(event.isSuccess()); } + + @Test + public void testProtectionMethodNamesMatchAuditSchema() { + assertEquals( + UserDataTransferProtectionMethod.TLS, + UserDataTransferProtectionMethod.fromTlsEnabled(true)); + assertEquals( + UserDataTransferProtectionMethod.NONE, + UserDataTransferProtectionMethod.fromTlsEnabled(false)); + } } From 85ff1e2b0210a14234e27cc965fa8e879e9d74b9 Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 2 Sep 2026 17:32:06 +0800 Subject: [PATCH 4/8] [Feature] Gate audit classification and filter snapshots --- .../UserDataTransferAuditClassifier.java | 18 ++++++- .../consensus/iot/IoTConsensusServerImpl.java | 27 +++++++--- .../iot/IoTConsensusServerImplTest.java | 36 +++++++++++++ .../DataNodeUserDataTransferAuditor.java | 15 +++++- .../db/consensus/DataRegionConsensusImpl.java | 15 +++++- .../plan/scheduler/AsyncPlanNodeSender.java | 3 +- .../DataNodeUserDataTransferAuditorTest.java | 7 +++ .../scheduler/AsyncPlanNodeSenderTest.java | 54 ++++++++++++++----- 8 files changed, 152 insertions(+), 23 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java index 952f0b2125759..8c8e418cb0816 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/UserDataTransferAuditClassifier.java @@ -25,7 +25,23 @@ @FunctionalInterface public interface UserDataTransferAuditClassifier { - UserDataTransferAuditClassifier NO_USER_DATA = (groupId, request) -> false; + UserDataTransferAuditClassifier NO_USER_DATA = + new UserDataTransferAuditClassifier() { + @Override + public boolean containsUserData(ConsensusGroupId groupId, IConsensusRequest request) { + return false; + } + + @Override + public boolean containsUserData(ConsensusGroupId groupId) { + return false; + } + }; boolean containsUserData(ConsensusGroupId groupId, IConsensusRequest request); + + /** Classifies a whole consensus group when the transfer has no individual request to inspect. */ + default boolean containsUserData(ConsensusGroupId groupId) { + return true; + } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index 179e04d95948e..4b769ff52d02f 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -428,6 +428,9 @@ public void takeSnapshot() throws ConsensusGroupModifyPeerException { public void transmitSnapshot(Peer targetPeer) throws ConsensusGroupModifyPeerException { File snapshotDir = new File(storageDir, newSnapshotDirName); List snapshotPaths = stateMachine.getSnapshotFiles(snapshotDir); + final boolean auditSnapshotTransfer = + shouldAuditSnapshotTransfer( + userDataTransferAuditHandler, userDataTransferAuditClassifier, thisNode.getGroupId()); long snapshotSizeSum = 0; for (File file : snapshotPaths) { snapshotSizeSum += file.length(); @@ -475,12 +478,12 @@ public void transmitSnapshot(Peer targetPeer) throws ConsensusGroupModifyPeerExc res = client.sendSnapshotFragment(req); recordSnapshotTransferAttempt( targetPeer, - req, + auditSnapshotTransfer, isSuccess(res.getStatus()), isSuccess(res.getStatus()) ? null : String.valueOf(res.getStatus().getCode()), null); } catch (Exception e) { - recordSnapshotTransferAttempt(targetPeer, req, false, null, e); + recordSnapshotTransferAttempt(targetPeer, auditSnapshotTransfer, false, null, e); throw e; } if (!isSuccess(res.getStatus())) { @@ -521,14 +524,14 @@ public void transmitSnapshot(Peer targetPeer) throws ConsensusGroupModifyPeerExc private void recordSnapshotTransferAttempt( Peer targetPeer, - TSendSnapshotFragmentReq request, + boolean auditSnapshotTransfer, boolean success, String errorCode, Throwable error) { + if (!auditSnapshotTransfer) { + return; + } try { - if (!userDataTransferAuditHandler.isEnabled()) { - return; - } userDataTransferAuditHandler.onAttempt( new UserDataTransferAuditEvent( thisNode.getEndpoint(), @@ -542,6 +545,18 @@ private void recordSnapshotTransferAttempt( } } + static boolean shouldAuditSnapshotTransfer( + UserDataTransferAuditHandler auditHandler, + UserDataTransferAuditClassifier auditClassifier, + ConsensusGroupId groupId) { + try { + return auditHandler.isEnabled() && auditClassifier.containsUserData(groupId); + } catch (RuntimeException ignored) { + // Audit classification must not affect snapshot transmission. + return false; + } + } + public void receiveSnapshotFragment( String snapshotId, String originalFilePath, ByteBuffer fileChunk, long fileOffset) throws ConsensusGroupModifyPeerException { diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImplTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImplTest.java index 24f8e8dc6bd01..cc33cf1573a4c 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImplTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImplTest.java @@ -20,10 +20,14 @@ package org.apache.iotdb.consensus.iot; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType; +import org.apache.iotdb.commons.request.IConsensusRequest; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.config.UserDataTransferAuditClassifier; import org.apache.iotdb.consensus.iot.util.TestStateMachine; import org.junit.Rule; @@ -44,6 +48,7 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class IoTConsensusServerImplTest { @@ -54,6 +59,37 @@ public class IoTConsensusServerImplTest { @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testSnapshotTransferAppliesGroupClassifier() { + final DataRegionId groupId = new DataRegionId(1); + final UserDataTransferAuditHandler enabledHandler = event -> {}; + final UserDataTransferAuditClassifier auditDatabaseClassifier = + new UserDataTransferAuditClassifier() { + @Override + public boolean containsUserData( + ConsensusGroupId ignoredGroupId, IConsensusRequest request) { + return false; + } + + @Override + public boolean containsUserData(ConsensusGroupId ignoredGroupId) { + return false; + } + }; + + assertTrue( + IoTConsensusServerImpl.shouldAuditSnapshotTransfer( + enabledHandler, (ignoredGroupId, request) -> false, groupId)); + assertFalse( + IoTConsensusServerImpl.shouldAuditSnapshotTransfer( + enabledHandler, auditDatabaseClassifier, groupId)); + assertFalse( + IoTConsensusServerImpl.shouldAuditSnapshotTransfer( + UserDataTransferAuditHandler.NO_OP, + UserDataTransferAuditClassifier.NO_USER_DATA, + groupId)); + } + /** * Verifies that configuration snapshots can be read while several writers concurrently add and * remove distinct peers. Every snapshot must remain duplicate-free and sorted, and the final diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java index d519dba836849..ad13725fc9546 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java @@ -87,8 +87,21 @@ public static boolean containsUserData( return dataRegion != null && containsUserData(dataRegion.getDatabaseName(), request); } + public static boolean containsUserData(ConsensusGroupId consensusGroupId) { + if (!(consensusGroupId instanceof DataRegionId)) { + return false; + } + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion((DataRegionId) consensusGroupId); + return dataRegion != null && containsUserData(dataRegion.getDatabaseName()); + } + + static boolean containsUserData(String database) { + return !Audit.isAuditDatabase(database); + } + static boolean containsUserData(String database, IConsensusRequest request) { - if (Audit.isAuditDatabase(database)) { + if (!containsUserData(database)) { return false; } try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index 218e0fbcc1c34..ba08b7379bcc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.memory.IMemoryBlock; import org.apache.iotdb.commons.memory.MemoryBlockType; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; +import org.apache.iotdb.commons.request.IConsensusRequest; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.IConsensus; @@ -70,6 +71,18 @@ public class DataRegionConsensusImpl { private static final Logger LOGGER = LoggerFactory.getLogger(DataRegionConsensusImpl.class); + private static final UserDataTransferAuditClassifier USER_DATA_TRANSFER_AUDIT_CLASSIFIER = + new UserDataTransferAuditClassifier() { + @Override + public boolean containsUserData(ConsensusGroupId groupId, IConsensusRequest request) { + return DataNodeUserDataTransferAuditor.containsUserData(groupId, request); + } + + @Override + public boolean containsUserData(ConsensusGroupId groupId) { + return DataNodeUserDataTransferAuditor.containsUserData(groupId); + } + }; private DataRegionConsensusImpl() { // do nothing @@ -152,7 +165,7 @@ private static ConsensusConfig buildConsensusConfig() { : UserDataTransferAuditHandler.NO_OP) .setUserDataTransferAuditClassifier( COMMON_CONF.isEnableAuditLog() - ? DataNodeUserDataTransferAuditor::containsUserData + ? USER_DATA_TRANSFER_AUDIT_CLASSIFIER : UserDataTransferAuditClassifier.NO_USER_DATA) .setStorageDir(CONF.getDataRegionConsensusDir()) .setRecvSnapshotDirs(Arrays.asList(CONF.getLocalDataDirs())) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java index 2346cfd637d9e..19c8c25fd9650 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java @@ -256,7 +256,8 @@ private static boolean containsUserData(FragmentInstance instance) { } static boolean containsUserData(PlanNode node, String username) { - return !User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME.equals(username) + return DataNodeUserDataTransferAuditor.isEnabled() + && !User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME.equals(username) && DataNodeUserDataTransferAuditor.containsInsertNode(node); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java index 9268f57b0c375..e0cc4ed1172c8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java @@ -33,6 +33,13 @@ public class DataNodeUserDataTransferAuditorTest { + @Test + public void testAuditDatabaseIsExcludedFromGroupTransferAudit() { + assertFalse(DataNodeUserDataTransferAuditor.containsUserData("__audit")); + assertFalse(DataNodeUserDataTransferAuditor.containsUserData("root.__audit")); + assertTrue(DataNodeUserDataTransferAuditor.containsUserData("root.sg")); + } + @Test public void testAuditDatabaseIsExcludedFromConsensusTransferAudit() { final InsertNode insertNode = mock(InsertNode.class); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java index 4ef63bca9d8a3..42fb38a719431 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.queryengine.plan.scheduler; import org.apache.iotdb.commons.auth.entity.User; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; @@ -30,24 +32,50 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class AsyncPlanNodeSenderTest { @Test public void testOnlyInsertPayloadIsClassifiedAsUserData() { - final PlanNode queryPlan = mock(PlanNode.class); - when(queryPlan.getChildren()).thenReturn(Collections.emptyList()); - assertFalse(AsyncPlanNodeSender.containsInsertNode(queryPlan)); - - final InsertNode insertNode = mock(InsertNode.class); - assertTrue(AsyncPlanNodeSender.containsInsertNode(insertNode)); - assertTrue(AsyncPlanNodeSender.containsUserData(insertNode, "root")); - assertFalse( - AsyncPlanNodeSender.containsUserData(insertNode, User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME)); - - final PlanNode wrapper = mock(PlanNode.class); - when(wrapper.getChildren()).thenReturn(Collections.singletonList(insertNode)); - assertTrue(AsyncPlanNodeSender.containsInsertNode(wrapper)); + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + final boolean auditLogEnabled = commonConfig.isEnableAuditLog(); + try { + commonConfig.setEnableAuditLog(true); + + final PlanNode queryPlan = mock(PlanNode.class); + when(queryPlan.getChildren()).thenReturn(Collections.emptyList()); + assertFalse(AsyncPlanNodeSender.containsInsertNode(queryPlan)); + + final InsertNode insertNode = mock(InsertNode.class); + assertTrue(AsyncPlanNodeSender.containsInsertNode(insertNode)); + assertTrue(AsyncPlanNodeSender.containsUserData(insertNode, "root")); + assertFalse( + AsyncPlanNodeSender.containsUserData( + insertNode, User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME)); + + final PlanNode wrapper = mock(PlanNode.class); + when(wrapper.getChildren()).thenReturn(Collections.singletonList(insertNode)); + assertTrue(AsyncPlanNodeSender.containsInsertNode(wrapper)); + } finally { + commonConfig.setEnableAuditLog(auditLogEnabled); + } + } + + @Test + public void testAuditDisabledSkipsPlanTraversal() { + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + final boolean auditLogEnabled = commonConfig.isEnableAuditLog(); + final PlanNode planNode = mock(PlanNode.class); + try { + commonConfig.setEnableAuditLog(false); + + assertFalse(AsyncPlanNodeSender.containsUserData(planNode, "root")); + verify(planNode, never()).getChildren(); + } finally { + commonConfig.setEnableAuditLog(auditLogEnabled); + } } } From 9e6cbee1b32ee10249f636254f4996385b59b86f Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 2 Sep 2026 19:14:22 +0800 Subject: [PATCH 5/8] [Feature] Address audit transfer review feedback --- .../consensus/i18n/IoTConsensusMessages.java | 3 ++ .../consensus/i18n/IoTConsensusMessages.java | 3 ++ .../consensus/iot/IoTConsensusServerImpl.java | 10 ++++- .../iot/client/DispatchLogHandler.java | 41 +++++++++++-------- .../iot/logdispatcher/LogDispatcher.java | 5 ++- .../DataNodeUserDataTransferAuditor.java | 23 +++-------- ...IoTConsensusV2TabletBatchEventHandler.java | 10 +++-- .../plan/scheduler/AsyncPlanNodeSender.java | 6 +-- .../dataregion/wal/node/WALNode.java | 16 +++++++- .../DataNodeUserDataTransferAuditorTest.java | 17 ++++++++ .../scheduler/AsyncPlanNodeSenderTest.java | 6 +-- .../wal/node/ConsensusReqReaderTest.java | 2 + 12 files changed, 96 insertions(+), 46 deletions(-) diff --git a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index 59e3567109b41..7f6efd07aebc5 100644 --- a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -333,5 +333,8 @@ private IoTConsensusMessages() {} public static final String LOG_RESERVED_ARG_BYTES_BATCH_ARG_ARG_CURRENT_TOTAL_USAGE_ARG_308AE9C2 = "Reserved {} bytes for batch {}-{}, current total usage {}"; public static final String LOG_ARG_FAILED_SEND_IDLE_WRITER_SAFE_TIME_BARRIER_ARG_STATUS_AE047EAD = "{}: Failed to send idle writer safe-time barrier to {}. status={}"; public static final String LOG_ARG_WRITE_OPERATION_FAILED_SEARCHINDEX_ARG_CODE_ARG_SUBSCRIPTIONQUEUES_ARG_THIS_ARG_F4B17576 = "{}: write operation failed. searchIndex: {}. Code: {}, subscriptionQueues: {}, this: {}"; + public static final String + LOG_FAILED_TO_RECORD_A_USER_DATA_TRANSFER_AUDIT_EVENT_CONSENSUS_REPLICATION_WILL_CONTINUE_F215E222 = + "Failed to record a user-data transfer audit event; consensus replication will continue."; } diff --git a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index 5164b57b9f581..7cc8ac622d194 100644 --- a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -331,5 +331,8 @@ private IoTConsensusMessages() {} public static final String LOG_RESERVED_ARG_BYTES_BATCH_ARG_ARG_CURRENT_TOTAL_USAGE_ARG_308AE9C2 = "预留 {} 字节给批次 {}-{}，当前总使用量 {}"; public static final String LOG_ARG_FAILED_SEND_IDLE_WRITER_SAFE_TIME_BARRIER_ARG_STATUS_AE047EAD = "{}：无法向 {} 发送 idle writer safe-time barrier。状态={}"; public static final String LOG_ARG_WRITE_OPERATION_FAILED_SEARCHINDEX_ARG_CODE_ARG_SUBSCRIPTIONQUEUES_ARG_THIS_ARG_F4B17576 = "{}：写入操作失败。searchIndex: {}。Code: {}，订阅队列：{}，当前对象：{}"; + public static final String + LOG_FAILED_TO_RECORD_A_USER_DATA_TRANSFER_AUDIT_EVENT_CONSENSUS_REPLICATION_WILL_CONTINUE_F215E222 = + "记录用户数据传送审计事件失败；Consensus 复制将继续。"; } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index 4b769ff52d02f..ebe7c69f83e0d 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -1045,10 +1045,18 @@ public IndexedConsensusRequest buildIndexedConsensusRequestForRemoteRequest( req.setRoutingEpoch(routingEpoch); req.setPhysicalTime(physicalTime); req.setNodeId(nodeId); - req.setContainsUserData(containsUserData(requests)); return req; } + public boolean containsUserData() { + try { + return userDataTransferAuditClassifier.containsUserData(thisNode.getGroupId()); + } catch (RuntimeException ignored) { + // Classification is advisory and must not affect consensus replication. + return false; + } + } + public boolean containsUserData(List requests) { for (IConsensusRequest request : requests) { try { diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java index e85b0241d7af0..87ffc413c15d9 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java @@ -45,7 +45,7 @@ public class DispatchLogHandler implements AsyncMethodCallback { - private final Logger logger = LoggerFactory.getLogger(DispatchLogHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DispatchLogHandler.class); private final LogDispatcherThread thread; private final Batch batch; @@ -67,14 +67,16 @@ public DispatchLogHandler( @Override public void onComplete(TSyncLogEntriesRes response) { - final TSStatus failedStatus = + // One batch RPC is one physical transfer attempt, so keep one representative error value in + // the minimum audit record instead of concatenating an unbounded number of response details. + final TSStatus firstFailedStatus = response.getStatuses().stream() .filter(status -> status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) .findFirst() .orElse(null); recordTransferAttempt( - failedStatus == null, - failedStatus == null ? null : String.valueOf(failedStatus.getCode()), + firstFailedStatus == null, + firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), null); if (response.getStatuses().stream() .anyMatch(status -> RetryUtils.needRetryForWrite(status.getCode()))) { @@ -86,14 +88,14 @@ public void onComplete(TSyncLogEntriesRes response) { String messages = String.join(", ", retryStatusMessages); if (++retryCount == 1) { - logger.warn( + LOGGER.warn( IoTConsensusMessages.CANNOT_SEND_TO_PEER, batch, thread.getPeer(), retryCount, messages); } else { - logger.debug( + LOGGER.debug( IoTConsensusMessages.CANNOT_SEND_TO_PEER, batch, thread.getPeer(), @@ -102,13 +104,13 @@ public void onComplete(TSyncLogEntriesRes response) { } sleepCorrespondingTimeAndRetryAsynchronous(); } else { - if (logger.isDebugEnabled()) { + if (LOGGER.isDebugEnabled()) { boolean containsError = response.getStatuses().stream() .anyMatch( status -> status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()); if (containsError) { - logger.debug( + LOGGER.debug( IoTConsensusMessages.SEND_COMPLETE_BUT_CONTAINS_ERROR, batch, thread.getPeer(), @@ -131,14 +133,14 @@ public void onError(Exception exception) { Throwable rootCause = ExceptionUtils.getRootCause(exception); final Throwable actualCause = rootCause == null ? exception : rootCause; if (retryCount == 1) { - logger.warn( + LOGGER.warn( IoTConsensusMessages.CANNOT_SEND_TO_PEER_ON_ERROR, batch, thread.getPeer(), retryCount, actualCause.toString()); } else { - logger.debug( + LOGGER.debug( IoTConsensusMessages.CANNOT_SEND_TO_PEER_ON_ERROR, batch, thread.getPeer(), @@ -148,7 +150,7 @@ public void onError(Exception exception) { // skip TApplicationException caused by follower if (actualCause instanceof TApplicationException) { completeBatch(batch); - logger.warn(IoTConsensusMessages.SKIP_RETRY_TAPPLICATION_EXCEPTION, batch); + LOGGER.warn(IoTConsensusMessages.SKIP_RETRY_TAPPLICATION_EXCEPTION, batch); logDispatcherThreadMetrics.recordSyncLogTimePerRequest(System.nanoTime() - createTime); return; } @@ -167,7 +169,7 @@ private void sleepCorrespondingTimeAndRetryAsynchronous() { .schedule( () -> { if (thread.isStopped()) { - logger.debug( + LOGGER.debug( IoTConsensusMessages.LOG_DISPATCHER_STOPPED_NO_RETRY, thread.getPeer(), batch, @@ -199,8 +201,8 @@ private void recordTransferAttempt(boolean success, String errorCode, Throwable success, errorCode, error); - } catch (RuntimeException ignored) { - // Audit recording must not affect consensus replication. + } catch (RuntimeException auditFailure) { + warnAuditFailure(auditFailure); } } @@ -225,8 +227,15 @@ static void recordTransferAttempt( protectionMethod, success, errorCode != null ? errorCode : error == null ? null : error.getClass().getName())); - } catch (RuntimeException ignored) { - // Audit recording must not affect consensus replication. + } catch (RuntimeException auditFailure) { + warnAuditFailure(auditFailure); } } + + private static void warnAuditFailure(RuntimeException auditFailure) { + LOGGER.warn( + IoTConsensusMessages + .LOG_FAILED_TO_RECORD_A_USER_DATA_TRANSFER_AUDIT_EVENT_CONSENSUS_REPLICATION_WILL_CONTINUE_F215E222, + auditFailure); + } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java index 4fc9089856e64..1ff6579e0ded0 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java @@ -651,6 +651,7 @@ private boolean constructBatchFromWAL(long currentIndex, long maxIndex, Batch lo currentIndex, maxIndex); boolean hasCorruptedData = false; + final boolean consensusGroupContainsUserData = impl.containsUserData(); // targetIndex is the index of request that we need to find long targetIndex = currentIndex; // Even if there is no WAL files, these code won't produce error. @@ -675,7 +676,9 @@ private boolean constructBatchFromWAL(long currentIndex, long maxIndex, Batch lo hasCorruptedData = true; } targetIndex = data.getSearchIndex() + 1; - data.setContainsUserData(impl.containsUserData(data.getRequests())); + // The WAL reader derives this bit directly from the entry type. Apply the group-level + // exclusion here without deserializing the request solely for audit classification. + data.setContainsUserData(consensusGroupContainsUserData && data.containsUserData()); data.buildSerializedRequests(); // construct request from wal TLogEntry logEntry = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java index ad13725fc9546..69672dbc74ea9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java @@ -27,15 +27,12 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.commons.request.IConsensusRequest; import org.apache.iotdb.commons.schema.table.Audit; -import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest; -import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; -import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import javax.annotation.Nullable; @@ -105,32 +102,22 @@ static boolean containsUserData(String database, IConsensusRequest request) { return false; } try { - final PlanNode planNode; - if (request instanceof PlanNode) { - planNode = (PlanNode) request; - } else if (request instanceof IoTConsensusRequest) { - planNode = WALEntry.deserializeForConsensus(request.serializeToByteBuffer().duplicate()); - } else if (request instanceof ByteBufferConsensusRequest) { - planNode = PlanNodeType.deserialize(request.serializeToByteBuffer().duplicate()); - } else { - return false; - } - return containsInsertNode(planNode); + return request instanceof PlanNode && containsUserData((PlanNode) request); } catch (RuntimeException ignored) { // Classification is advisory and must not affect consensus replication. return false; } } - public static boolean containsInsertNode(PlanNode node) { - if (node instanceof InsertNode) { + public static boolean containsUserData(PlanNode node) { + if (node instanceof InsertNode || node instanceof ObjectNode) { return true; } if (node.getChildren() == null) { return false; } for (PlanNode child : node.getChildren()) { - if (containsInsertNode(child)) { + if (containsUserData(child)) { return true; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java index 2ccefdeb2ed9a..bd415f53eea05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java @@ -85,14 +85,16 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { response.getBatchResps().stream() .map(TIoTConsensusV2TransferResp::getStatus) .collect(Collectors.toList()); - final TSStatus failedStatus = + // The batch RPC is one physical transfer attempt. Keep one representative error value in + // the minimum audit record instead of concatenating an unbounded number of response details. + final TSStatus firstFailedStatus = status.stream() .filter(tsStatus -> tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) .findFirst() .orElse(null); connector.recordUserDataTransferAudit( - failedStatus == null, - failedStatus == null ? null : String.valueOf(failedStatus.getCode()), + firstFailedStatus == null, + firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), null); transferAuditRecorded = true; @@ -129,6 +131,8 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { @Override public void onError(final Exception exception) { + // A retry is sent through a new handler and produces its own audit event. This guard only + // prevents a post-response processing exception from recording the same attempt twice. if (!transferAuditRecorded) { connector.recordUserDataTransferAudit(false, null, exception); transferAuditRecorded = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java index 19c8c25fd9650..60a1739720e39 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSender.java @@ -258,10 +258,10 @@ private static boolean containsUserData(FragmentInstance instance) { static boolean containsUserData(PlanNode node, String username) { return DataNodeUserDataTransferAuditor.isEnabled() && !User.BUILTIN_INTERNAL_AUDIT_LOG_USERNAME.equals(username) - && DataNodeUserDataTransferAuditor.containsInsertNode(node); + && DataNodeUserDataTransferAuditor.containsUserData(node); } - static boolean containsInsertNode(PlanNode node) { - return DataNodeUserDataTransferAuditor.containsInsertNode(node); + static boolean containsUserData(PlanNode node) { + return DataNodeUserDataTransferAuditor.containsUserData(node); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 03438b60707d4..0e66509d35099 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -766,6 +766,7 @@ public boolean hasNext() { AtomicLong currentEntryLocalSeq = new AtomicLong(-1); AtomicLong currentEntryPhysicalTime = new AtomicLong(0); AtomicLong currentEntryNodeId = new AtomicLong(-1); + AtomicBoolean currentEntryContainsUserData = new AtomicBoolean(false); long memorySize = 0; @@ -779,10 +780,12 @@ public boolean hasNext() { (localSeq >= 0) ? new IndexedConsensusRequest(nextSearchIndex, localSeq, tmpNodes.get()) : new IndexedConsensusRequest(nextSearchIndex, tmpNodes.get()); - req.setPhysicalTime(currentEntryPhysicalTime.get()) + req.setContainsUserData(currentEntryContainsUserData.get()) + .setPhysicalTime(currentEntryPhysicalTime.get()) .setNodeId((int) currentEntryNodeId.get()); insertNodes.add(req); tmpNodes.set(new ArrayList<>()); + currentEntryContainsUserData.set(false); nextSearchIndex++; if (notFirstFile.get()) { hasCollectedSufficientData.set(true); @@ -819,6 +822,8 @@ public boolean hasNext() { currentEntryLocalSeq.set(walByteBufReader.getCurrentEntryLocalSeq()); currentEntryPhysicalTime.set(walByteBufReader.getCurrentEntryPhysicalTime()); currentEntryNodeId.set(walByteBufReader.getCurrentEntryNodeId()); + currentEntryContainsUserData.set( + currentEntryContainsUserData.get() || containsUserData(type)); if (type == WALEntryType.OBJECT_FILE_NODE) { WALEntry walEntry = WALEntry.deserialize( @@ -854,6 +859,8 @@ public boolean hasNext() { currentEntryLocalSeq.set(walByteBufReader.getCurrentEntryLocalSeq()); currentEntryPhysicalTime.set(walByteBufReader.getCurrentEntryPhysicalTime()); currentEntryNodeId.set(walByteBufReader.getCurrentEntryNodeId()); + currentEntryContainsUserData.set( + currentEntryContainsUserData.get() || containsUserData(type)); if (type == WALEntryType.OBJECT_FILE_NODE) { WALEntry walEntry = WALEntry.deserialize( @@ -909,6 +916,13 @@ public boolean hasNext() { return false; } + private boolean containsUserData(WALEntryType type) { + return type == WALEntryType.INSERT_ROW_NODE + || type == WALEntryType.INSERT_TABLET_NODE + || type == WALEntryType.INSERT_ROWS_NODE + || type == WALEntryType.OBJECT_FILE_NODE; + } + @Override public IndexedConsensusRequest next() { if (itr == null && !hasNext()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java index e0cc4ed1172c8..0361f304df75d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java @@ -20,7 +20,9 @@ package org.apache.iotdb.db.audit; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.request.IConsensusRequest; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.junit.Test; @@ -29,6 +31,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class DataNodeUserDataTransferAuditorTest { @@ -56,4 +60,17 @@ public void testNonInsertConsensusRequestIsExcluded() { assertFalse(DataNodeUserDataTransferAuditor.containsUserData("root.sg", planNode)); } + + @Test + public void testClassificationDoesNotDeserializeConsensusRequest() { + final IConsensusRequest request = mock(IConsensusRequest.class); + + assertFalse(DataNodeUserDataTransferAuditor.containsUserData("root.sg", request)); + verify(request, never()).serializeToByteBuffer(); + } + + @Test + public void testObjectFileNodeContainsUserData() { + assertTrue(DataNodeUserDataTransferAuditor.containsUserData(mock(ObjectNode.class))); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java index 42fb38a719431..a841c35d78f4b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/AsyncPlanNodeSenderTest.java @@ -47,10 +47,10 @@ public void testOnlyInsertPayloadIsClassifiedAsUserData() { final PlanNode queryPlan = mock(PlanNode.class); when(queryPlan.getChildren()).thenReturn(Collections.emptyList()); - assertFalse(AsyncPlanNodeSender.containsInsertNode(queryPlan)); + assertFalse(AsyncPlanNodeSender.containsUserData(queryPlan)); final InsertNode insertNode = mock(InsertNode.class); - assertTrue(AsyncPlanNodeSender.containsInsertNode(insertNode)); + assertTrue(AsyncPlanNodeSender.containsUserData(insertNode)); assertTrue(AsyncPlanNodeSender.containsUserData(insertNode, "root")); assertFalse( AsyncPlanNodeSender.containsUserData( @@ -58,7 +58,7 @@ public void testOnlyInsertPayloadIsClassifiedAsUserData() { final PlanNode wrapper = mock(PlanNode.class); when(wrapper.getChildren()).thenReturn(Collections.singletonList(insertNode)); - assertTrue(AsyncPlanNodeSender.containsInsertNode(wrapper)); + assertTrue(AsyncPlanNodeSender.containsUserData(wrapper)); } finally { commonConfig.setEnableAuditLog(auditLogEnabled); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/ConsensusReqReaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/ConsensusReqReaderTest.java index e1a48eb73f009..1bd1c2c19f214 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/ConsensusReqReaderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/ConsensusReqReaderTest.java @@ -314,6 +314,7 @@ public void testReqIteratorCarriesWriterMetadata() throws Exception { Assert.assertEquals(1L, request.getSearchIndex()); Assert.assertEquals(123456789L, request.getPhysicalTime()); Assert.assertEquals(7, request.getNodeId()); + Assert.assertTrue(request.containsUserData()); } @Test @@ -514,6 +515,7 @@ public void scenario02TestGetReqIterator01() throws Exception { PlanNode planNode; Assert.assertTrue(iterator.hasNext()); request = iterator.next(); + Assert.assertFalse(request.containsUserData()); Assert.assertEquals(1, request.getRequests().size()); for (IConsensusRequest innerRequest : request.getRequests()) { planNode = WALEntry.deserializeForConsensus(innerRequest.serializeToByteBuffer()); From fc06291cbc7cb20c59fc91f1fb3700bd4e64eef1 Mon Sep 17 00:00:00 2001 From: HTHou Date: Thu, 3 Sep 2026 09:58:07 +0800 Subject: [PATCH 6/8] [Feature] Reduce disabled transfer audit overhead --- .../consensus/iot/IoTConsensusServerImpl.java | 22 +++--- .../iot/client/DispatchLogHandler.java | 71 +++++++++++++++---- .../iot/client/DispatchLogHandlerTest.java | 33 +++++++++ .../IoTConsensusV2AsyncSink.java | 13 ++-- .../IoTConsensusV2SyncSink.java | 28 +++++--- ...IoTConsensusV2TabletBatchEventHandler.java | 28 ++++---- 6 files changed, 148 insertions(+), 47 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index ebe7c69f83e0d..6e86ec72a3e3b 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -1027,8 +1027,9 @@ public IndexedConsensusRequest buildIndexedConsensusRequestForLocalRequest( new IoTProgressIndex(thisNode.getNodeId(), searchIndex.get() + 1); ((ComparableConsensusRequest) request).setProgressIndex(iotProgressIndex); } - return new IndexedConsensusRequest(searchIndex.get() + 1, Collections.singletonList(request)) - .setContainsUserData(containsUserData(Collections.singletonList(request))) + final List requests = Collections.singletonList(request); + return new IndexedConsensusRequest(searchIndex.get() + 1, requests) + .setContainsUserData(containsUserData(request)) .setPhysicalTime(assignPhysicalTimeInMs()) .setNodeId(thisNode.getNodeId()); } @@ -1059,17 +1060,22 @@ public boolean containsUserData() { public boolean containsUserData(List requests) { for (IConsensusRequest request : requests) { - try { - if (userDataTransferAuditClassifier.containsUserData(thisNode.getGroupId(), request)) { - return true; - } - } catch (RuntimeException ignored) { - // Classification is advisory and must not affect consensus replication. + if (containsUserData(request)) { + return true; } } return false; } + private boolean containsUserData(IConsensusRequest request) { + try { + return userDataTransferAuditClassifier.containsUserData(thisNode.getGroupId(), request); + } catch (RuntimeException ignored) { + // Classification is advisory and must not affect consensus replication. + return false; + } + } + public TSStatus syncIdleWriterSafeTimeBarrierToPeer(final Peer targetPeer) { final long safePhysicalTime = assignPhysicalTimeInMs(); final long safeLocalSeq = searchIndex.get(); diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java index 87ffc413c15d9..615ecd6b07c46 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java @@ -67,17 +67,7 @@ public DispatchLogHandler( @Override public void onComplete(TSyncLogEntriesRes response) { - // One batch RPC is one physical transfer attempt, so keep one representative error value in - // the minimum audit record instead of concatenating an unbounded number of response details. - final TSStatus firstFailedStatus = - response.getStatuses().stream() - .filter(status -> status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) - .findFirst() - .orElse(null); - recordTransferAttempt( - firstFailedStatus == null, - firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), - null); + recordTransferAttempt(response); if (response.getStatuses().stream() .anyMatch(status -> RetryUtils.needRetryForWrite(status.getCode()))) { List retryStatusMessages = @@ -191,8 +181,13 @@ private void completeBatch(Batch batch) { private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { try { + final UserDataTransferAuditHandler auditHandler = + thread.getImpl().getUserDataTransferAuditHandler(); + if (!batch.containsUserData() || !auditHandler.isEnabled()) { + return; + } recordTransferAttempt( - thread.getImpl().getUserDataTransferAuditHandler(), + auditHandler, batch, thread.getImpl().getThisNode().getEndpoint(), thread.getPeer().getEndpoint(), @@ -206,6 +201,58 @@ private void recordTransferAttempt(boolean success, String errorCode, Throwable } } + private void recordTransferAttempt(TSyncLogEntriesRes response) { + try { + final UserDataTransferAuditHandler auditHandler = + thread.getImpl().getUserDataTransferAuditHandler(); + if (!batch.containsUserData() || !auditHandler.isEnabled()) { + return; + } + recordTransferAttempt( + auditHandler, + batch, + thread.getImpl().getThisNode().getEndpoint(), + thread.getPeer().getEndpoint(), + UserDataTransferProtectionMethod.fromTlsEnabled( + thread.getConfig().getRpc().isEnableSSL()), + response); + } catch (RuntimeException auditFailure) { + warnAuditFailure(auditFailure); + } + } + + static void recordTransferAttempt( + UserDataTransferAuditHandler auditHandler, + Batch batch, + TEndPoint source, + TEndPoint target, + UserDataTransferProtectionMethod protectionMethod, + TSyncLogEntriesRes response) { + try { + if (!batch.containsUserData() || !auditHandler.isEnabled()) { + return; + } + // One batch RPC is one physical transfer attempt, so keep one representative error value in + // the minimum audit record instead of concatenating an unbounded number of response details. + final TSStatus firstFailedStatus = + response.getStatuses().stream() + .filter(status -> status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .findFirst() + .orElse(null); + recordTransferAttempt( + auditHandler, + batch, + source, + target, + protectionMethod, + firstFailedStatus == null, + firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), + null); + } catch (RuntimeException auditFailure) { + warnAuditFailure(auditFailure); + } + } + static void recordTransferAttempt( UserDataTransferAuditHandler auditHandler, Batch batch, diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java index 3d39c89bd7e03..13736903902a9 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java @@ -20,18 +20,21 @@ package org.apache.iotdb.consensus.iot.client; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; import org.apache.iotdb.consensus.config.IoTConsensusConfig; import org.apache.iotdb.consensus.iot.logdispatcher.Batch; import org.apache.iotdb.consensus.iot.thrift.TLogEntry; +import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes; import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -96,6 +99,36 @@ public void testSkipsBatchWithoutUserData() { assertTrue(events.isEmpty()); } + @Test + public void testSkipDoesNotInspectResponseStatuses() { + final AtomicInteger statusAccessCount = new AtomicInteger(); + final TSyncLogEntriesRes response = + new TSyncLogEntriesRes() { + @Override + public List getStatuses() { + statusAccessCount.incrementAndGet(); + return super.getStatuses(); + } + }; + + DispatchLogHandler.recordTransferAttempt( + event -> {}, + createBatch(false), + SOURCE, + TARGET, + UserDataTransferProtectionMethod.NONE, + response); + DispatchLogHandler.recordTransferAttempt( + UserDataTransferAuditHandler.NO_OP, + createBatch(true), + SOURCE, + TARGET, + UserDataTransferProtectionMethod.NONE, + response); + + assertEquals(0, statusAccessCount.get()); + } + @Test public void testAuditHandlerFailureDoesNotEscape() { DispatchLogHandler.recordTransferAttempt( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java index 99bd0ed401796..d43977d31350f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java @@ -110,6 +110,8 @@ public class IoTConsensusV2AsyncSink extends IoTDBSink implements ConsensusPipeS private ScheduledExecutorService backgroundTaskService; private final AtomicBoolean isClosed = new AtomicBoolean(false); private final int thisDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + private final TEndPoint localEndPoint = + new TEndPoint(IOTDB_CONFIG.getInternalAddress(), IOTDB_CONFIG.getDataRegionConsensusPort()); private IoTConsensusV2SinkMetrics iotConsensusV2SinkMetrics; private String consensusPipeName; private int consensusGroupId; @@ -747,11 +749,14 @@ public TEndPoint getFollowerUrl() { return nodeUrls.get(0); } + public boolean isUserDataTransferAuditEnabled() { + return DataNodeUserDataTransferAuditor.isEnabled(); + } + public void recordUserDataTransferAudit(boolean success, String errorCode, Throwable error) { - final TEndPoint localEndPoint = - new TEndPoint( - IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), - IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusPort()); + if (!isUserDataTransferAuditEnabled()) { + return; + } DataNodeUserDataTransferAuditor.record( localEndPoint, localEndPoint, getFollowerUrl(), success, errorCode, error); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java index a8a8dcc8d126e..2b3f194d7d3a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java @@ -89,6 +89,7 @@ public class IoTConsensusV2SyncSink extends IoTDBSink { private final int thisDataNodeId; private final int consensusGroupId; private final IoTConsensusV2SinkMetrics iotConsensusV2SinkMetrics; + private final TEndPoint localEndPoint; private IoTConsensusV2SyncBatchReqBuilder tabletBatchBuilder; public IoTConsensusV2SyncSink( @@ -102,6 +103,10 @@ public IoTConsensusV2SyncSink( this.peers = peers; this.consensusGroupId = consensusGroupId; this.thisDataNodeId = thisDataNodeId; + this.localEndPoint = + new TEndPoint( + IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), + IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusPort()); this.syncRetryClientManager = IoTV2GlobalComponentContainer.getInstance().getGlobalSyncClientManager(); this.iotConsensusV2SinkMetrics = iotConsensusV2SinkMetrics; @@ -213,13 +218,15 @@ private void doTransfer() { resp.getBatchResps().stream() .map(TIoTConsensusV2TransferResp::getStatus) .collect(Collectors.toList()); - final TSStatus failedStatus = - statusList.stream().filter(status -> !isSuccessful(status)).findFirst().orElse(null); - recordTransferAttempt( - failedStatus == null, - failedStatus == null ? null : String.valueOf(failedStatus.getCode()), - null); - transferAttemptRecorded = true; + if (DataNodeUserDataTransferAuditor.isEnabled()) { + final TSStatus failedStatus = + statusList.stream().filter(status -> !isSuccessful(status)).findFirst().orElse(null); + recordTransferAttempt( + failedStatus == null, + failedStatus == null ? null : String.valueOf(failedStatus.getCode()), + null); + transferAttemptRecorded = true; + } // TODO(support batch): handle retry logic // Only handle the failed statuses to avoid string format performance overhead @@ -575,10 +582,9 @@ private static boolean isSuccessful(TSStatus status) { } private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { - final TEndPoint localEndPoint = - new TEndPoint( - IoTDBDescriptor.getInstance().getConfig().getInternalAddress(), - IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusPort()); + if (!DataNodeUserDataTransferAuditor.isEnabled()) { + return; + } DataNodeUserDataTransferAuditor.record( localEndPoint, localEndPoint, getFollowerUrl(), success, errorCode, error); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java index bd415f53eea05..a0222de06e25a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java @@ -85,18 +85,22 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { response.getBatchResps().stream() .map(TIoTConsensusV2TransferResp::getStatus) .collect(Collectors.toList()); - // The batch RPC is one physical transfer attempt. Keep one representative error value in - // the minimum audit record instead of concatenating an unbounded number of response details. - final TSStatus firstFailedStatus = - status.stream() - .filter(tsStatus -> tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) - .findFirst() - .orElse(null); - connector.recordUserDataTransferAudit( - firstFailedStatus == null, - firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), - null); - transferAuditRecorded = true; + if (connector.isUserDataTransferAuditEnabled()) { + // The batch RPC is one physical transfer attempt. Keep one representative error value in + // the minimum audit record instead of concatenating an unbounded number of response + // details. + final TSStatus firstFailedStatus = + status.stream() + .filter( + tsStatus -> tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .findFirst() + .orElse(null); + connector.recordUserDataTransferAudit( + firstFailedStatus == null, + firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), + null); + transferAuditRecorded = true; + } if (status.stream() .anyMatch( From 8e796f4cd2123c2af4b4dd9853d15e8dc6355c89 Mon Sep 17 00:00:00 2001 From: HTHou Date: Thu, 3 Sep 2026 10:32:53 +0800 Subject: [PATCH 7/8] [Feature] Exclude audit database from IoTConsensusV2 audit --- .../DataNodeUserDataTransferAuditor.java | 9 +++ .../IoTConsensusV2AsyncSink.java | 5 +- .../IoTConsensusV2SyncSink.java | 11 +++- .../DataNodeUserDataTransferAuditorTest.java | 60 +++++++++++++++++++ 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java index 69672dbc74ea9..0ee38058066fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditor.java @@ -46,6 +46,15 @@ public static boolean isEnabled() { return COMMON_CONFIG.isEnableAuditLog(); } + public static boolean isEnabledFor(ConsensusGroupId consensusGroupId) { + try { + return isEnabled() && containsUserData(consensusGroupId); + } catch (RuntimeException ignored) { + // Classification is advisory and must not affect user data transfer. + return false; + } + } + public static void record( TEndPoint initiator, TEndPoint source, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java index d43977d31350f..1b30e79c326c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.iotv2.container.IoTV2GlobalComponentContainer; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkRetryTimesConfigurableException; @@ -112,6 +113,7 @@ public class IoTConsensusV2AsyncSink extends IoTDBSink implements ConsensusPipeS private final int thisDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); private final TEndPoint localEndPoint = new TEndPoint(IOTDB_CONFIG.getInternalAddress(), IOTDB_CONFIG.getDataRegionConsensusPort()); + private DataRegionId dataRegionId; private IoTConsensusV2SinkMetrics iotConsensusV2SinkMetrics; private String consensusPipeName; private int consensusGroupId; @@ -141,6 +143,7 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati // Get consensusGroupId from parameters passed by IoTConsensusV2Impl consensusGroupId = parameters.getInt(CONNECTOR_CONSENSUS_GROUP_ID_KEY); + dataRegionId = new DataRegionId(consensusGroupId); // Get consensusPipeName from parameters passed by IoTConsensusV2Impl consensusPipeName = parameters.getString(CONNECTOR_CONSENSUS_PIPE_NAME); @@ -750,7 +753,7 @@ public TEndPoint getFollowerUrl() { } public boolean isUserDataTransferAuditEnabled() { - return DataNodeUserDataTransferAuditor.isEnabled(); + return dataRegionId != null && DataNodeUserDataTransferAuditor.isEnabledFor(dataRegionId); } public void recordUserDataTransferAudit(boolean success, String errorCode, Throwable error) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java index 2b3f194d7d3a1..9d6c3697091be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java @@ -25,6 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncIoTConsensusV2ServiceClient; +import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.iotv2.container.IoTV2GlobalComponentContainer; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkRetryTimesConfigurableException; @@ -90,6 +91,7 @@ public class IoTConsensusV2SyncSink extends IoTDBSink { private final int consensusGroupId; private final IoTConsensusV2SinkMetrics iotConsensusV2SinkMetrics; private final TEndPoint localEndPoint; + private final DataRegionId dataRegionId; private IoTConsensusV2SyncBatchReqBuilder tabletBatchBuilder; public IoTConsensusV2SyncSink( @@ -102,6 +104,7 @@ public IoTConsensusV2SyncSink( // retain the implementation of list to cope with possible future expansion this.peers = peers; this.consensusGroupId = consensusGroupId; + this.dataRegionId = new DataRegionId(consensusGroupId); this.thisDataNodeId = thisDataNodeId; this.localEndPoint = new TEndPoint( @@ -218,7 +221,7 @@ private void doTransfer() { resp.getBatchResps().stream() .map(TIoTConsensusV2TransferResp::getStatus) .collect(Collectors.toList()); - if (DataNodeUserDataTransferAuditor.isEnabled()) { + if (isUserDataTransferAuditEnabled()) { final TSStatus failedStatus = statusList.stream().filter(status -> !isSuccessful(status)).findFirst().orElse(null); recordTransferAttempt( @@ -582,13 +585,17 @@ private static boolean isSuccessful(TSStatus status) { } private void recordTransferAttempt(boolean success, String errorCode, Throwable error) { - if (!DataNodeUserDataTransferAuditor.isEnabled()) { + if (!isUserDataTransferAuditEnabled()) { return; } DataNodeUserDataTransferAuditor.record( localEndPoint, localEndPoint, getFollowerUrl(), success, errorCode, error); } + private boolean isUserDataTransferAuditEnabled() { + return DataNodeUserDataTransferAuditor.isEnabledFor(dataRegionId); + } + // synchronized to avoid close connector when transfer event @Override public synchronized void close() { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java index 0361f304df75d..0cba74868b65c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/audit/DataNodeUserDataTransferAuditorTest.java @@ -19,12 +19,24 @@ package org.apache.iotdb.db.audit; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.request.IConsensusRequest; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import java.util.Collections; @@ -35,8 +47,24 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) +@RunWith(PowerMockRunner.class) +@PrepareForTest(StorageEngine.class) public class DataNodeUserDataTransferAuditorTest { + private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + private boolean auditLogEnabled; + + @Before + public void setUp() { + auditLogEnabled = commonConfig.isEnableAuditLog(); + } + + @After + public void tearDown() { + commonConfig.setEnableAuditLog(auditLogEnabled); + } + @Test public void testAuditDatabaseIsExcludedFromGroupTransferAudit() { assertFalse(DataNodeUserDataTransferAuditor.containsUserData("__audit")); @@ -44,6 +72,38 @@ public void testAuditDatabaseIsExcludedFromGroupTransferAudit() { assertTrue(DataNodeUserDataTransferAuditor.containsUserData("root.sg")); } + @Test + public void testIoTConsensusV2AuditGateExcludesAuditDatabase() { + final StorageEngine storageEngine = mock(StorageEngine.class); + final DataRegion auditDataRegion = mock(DataRegion.class); + final DataRegion userDataRegion = mock(DataRegion.class); + final DataRegionId auditDataRegionId = new DataRegionId(1); + final DataRegionId userDataRegionId = new DataRegionId(2); + PowerMockito.mockStatic(StorageEngine.class); + PowerMockito.when(StorageEngine.getInstance()).thenReturn(storageEngine); + when(storageEngine.getDataRegion(auditDataRegionId)).thenReturn(auditDataRegion); + when(storageEngine.getDataRegion(userDataRegionId)).thenReturn(userDataRegion); + when(auditDataRegion.getDatabaseName()).thenReturn("root.__audit"); + when(userDataRegion.getDatabaseName()).thenReturn("root.sg"); + + commonConfig.setEnableAuditLog(true); + + assertFalse(DataNodeUserDataTransferAuditor.isEnabledFor(auditDataRegionId)); + assertTrue(DataNodeUserDataTransferAuditor.isEnabledFor(userDataRegionId)); + } + + @Test + public void testIoTConsensusV2AuditGateShortCircuitsWhenDisabled() { + final StorageEngine storageEngine = mock(StorageEngine.class); + PowerMockito.mockStatic(StorageEngine.class); + PowerMockito.when(StorageEngine.getInstance()).thenReturn(storageEngine); + commonConfig.setEnableAuditLog(false); + + assertFalse(DataNodeUserDataTransferAuditor.isEnabledFor(new DataRegionId(1))); + + verify(storageEngine, never()).getDataRegion(new DataRegionId(1)); + } + @Test public void testAuditDatabaseIsExcludedFromConsensusTransferAudit() { final InsertNode insertNode = mock(InsertNode.class); From b46dd272269e2fa06684c6b6315405b901c48ff2 Mon Sep 17 00:00:00 2001 From: HTHou Date: Thu, 3 Sep 2026 10:41:05 +0800 Subject: [PATCH 8/8] [Feature] Avoid duplicate IoTConsensusV2 audit classification --- .../IoTConsensusV2AsyncSink.java | 26 +++++++++++++++++++ .../IoTConsensusV2SyncSink.java | 7 ++++- ...IoTConsensusV2TabletBatchEventHandler.java | 17 +----------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java index 1b30e79c326c4..327309005d8f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2AsyncSink.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.consensus.ConsensusGroupId; @@ -65,6 +66,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +75,7 @@ import java.io.IOException; import java.util.Comparator; import java.util.Iterator; +import java.util.List; import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; @@ -760,6 +763,29 @@ public void recordUserDataTransferAudit(boolean success, String errorCode, Throw if (!isUserDataTransferAuditEnabled()) { return; } + recordUserDataTransferAuditWithoutGroupCheck(success, errorCode, error); + } + + public boolean recordUserDataTransferAudit(List statuses) { + if (!isUserDataTransferAuditEnabled()) { + return false; + } + // The batch RPC is one physical transfer attempt. Keep one representative error value in the + // minimum audit record instead of concatenating an unbounded number of response details. + final TSStatus firstFailedStatus = + statuses.stream() + .filter(status -> status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .findFirst() + .orElse(null); + recordUserDataTransferAuditWithoutGroupCheck( + firstFailedStatus == null, + firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), + null); + return true; + } + + private void recordUserDataTransferAuditWithoutGroupCheck( + boolean success, String errorCode, Throwable error) { DataNodeUserDataTransferAuditor.record( localEndPoint, localEndPoint, getFollowerUrl(), success, errorCode, error); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java index 9d6c3697091be..02b8f7cd15d75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/IoTConsensusV2SyncSink.java @@ -224,7 +224,7 @@ private void doTransfer() { if (isUserDataTransferAuditEnabled()) { final TSStatus failedStatus = statusList.stream().filter(status -> !isSuccessful(status)).findFirst().orElse(null); - recordTransferAttempt( + recordTransferAttemptWithoutGroupCheck( failedStatus == null, failedStatus == null ? null : String.valueOf(failedStatus.getCode()), null); @@ -588,6 +588,11 @@ private void recordTransferAttempt(boolean success, String errorCode, Throwable if (!isUserDataTransferAuditEnabled()) { return; } + recordTransferAttemptWithoutGroupCheck(success, errorCode, error); + } + + private void recordTransferAttemptWithoutGroupCheck( + boolean success, String errorCode, Throwable error) { DataNodeUserDataTransferAuditor.record( localEndPoint, localEndPoint, getFollowerUrl(), success, errorCode, error); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java index a0222de06e25a..527fca89001e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/iotconsensusv2/handler/IoTConsensusV2TabletBatchEventHandler.java @@ -85,22 +85,7 @@ public void onComplete(final TIoTConsensusV2BatchTransferResp response) { response.getBatchResps().stream() .map(TIoTConsensusV2TransferResp::getStatus) .collect(Collectors.toList()); - if (connector.isUserDataTransferAuditEnabled()) { - // The batch RPC is one physical transfer attempt. Keep one representative error value in - // the minimum audit record instead of concatenating an unbounded number of response - // details. - final TSStatus firstFailedStatus = - status.stream() - .filter( - tsStatus -> tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) - .findFirst() - .orElse(null); - connector.recordUserDataTransferAudit( - firstFailedStatus == null, - firstFailedStatus == null ? null : String.valueOf(firstFailedStatus.getCode()), - null); - transferAuditRecorded = true; - } + transferAuditRecorded = connector.recordUserDataTransferAudit(status); if (status.stream() .anyMatch(