From 4986f07b5c2f071e871ea809be0823e15c450943 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 29 Oct 2026 19:35:34 +0100 Subject: [PATCH 1/4] docs: a microphone track hands no audio to sinks WebRTC's LocalAudioSource ignores AddSink, so VoiceActivityExample attached to a microphone track hears nothing. Say which tracks feed a sink, and point to AudioRecorder for the microphone. --- docs/tools/audio/voice-activity-detector.md | 11 ++++++++--- .../onvoid/webrtc/examples/VoiceActivityExample.java | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/tools/audio/voice-activity-detector.md b/docs/tools/audio/voice-activity-detector.md index b658c84b4..4e9aa8c17 100644 --- a/docs/tools/audio/voice-activity-detector.md +++ b/docs/tools/audio/voice-activity-detector.md @@ -147,9 +147,9 @@ peer.onTrack(track -> { }); ``` -The client API has no audio subscription on a local track. To detect speech at -the microphone, attach the adapter as an `AudioTrackSink` to an `AudioTrack` of -the native API: +An `AudioTrack` of the native API takes the adapter as an `AudioTrackSink`, as +long as the track's source hands its audio to sinks. A remote track and a track +over a `CustomAudioSource` do: ```java VoiceActivityExample sink = new VoiceActivityExample(speech -> { @@ -165,6 +165,11 @@ finally { } ``` +A microphone track does not: WebRTC passes captured audio from the audio device +module straight to the senders, and the source of the track hands nothing to +sinks. An [`AudioRecorder`](/tools/audio/audio-recorder) captures a microphone +on its own, apart from any peer connection. + The adapter invokes the consumer on the thread that delivers the audio: a thread of the `onAudio` subscription, or the audio callback thread of a sink. Keep work there short. A subscription that falls behind loses the oldest audio beyond its diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java index 90c03fbe6..53b4c6bb9 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java @@ -31,8 +31,9 @@ /** * With the client API, subscribe it to a remote track with * {@link RemoteAudio#onAudio(Consumer)}, then close the {@link Subscription} - * before close(). An AudioTrack of the native API, such as a microphone's, - * takes it with addSink(), then removeSink() before close(). + * before close(). An AudioTrack of the native API takes it with addSink(), + * then removeSink() before close(), if its source hands audio to sinks: that + * of a remote track or a CustomAudioSource does, a microphone's does not. * Buffers arbitrary chunk lengths into 10 ms frames and converts interleaved * signed 16-bit PCM to 16 kHz mono, including input at 44.1 kHz. The consumer runs * on the thread that delivers the audio and should return promptly. It receives raw From 77f2045054971a43c1e4ce2e642735eb239b4bf6 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 29 Oct 2026 20:03:47 +0100 Subject: [PATCH 2/4] feat: the audio a local track sends, in the client API LocalAudio.onAudio hands pushed audio as pushed, and a microphone's audio after audio processing, as the senders send it. Every AudioProcessing now carries a capture post-processor that hands its output to a sink, through AudioProcessing.setCapturedAudioSink and PeerConnectionFactory.setCapturedAudioSink. --- docs/guide/client-api.md | 9 + .../main/cpp/include/JNI_AudioProcessing.h | 8 + .../cpp/include/JNI_PeerConnectionFactory.h | 8 + .../cpp/include/media/audio/AudioCaptureTap.h | 63 +++++ .../src/main/cpp/src/JNI_AudioProcessing.cpp | 33 ++- .../cpp/src/JNI_PeerConnectionFactory.cpp | 53 ++++- .../cpp/src/media/audio/AudioCaptureTap.cpp | 102 ++++++++ .../onvoid/webrtc/PeerConnectionFactory.java | 24 ++ .../onvoid/webrtc/client/CapturedAudio.java | 64 +++++ .../onvoid/webrtc/client/CustomAudioImpl.java | 2 +- .../dev/onvoid/webrtc/client/LocalAudio.java | 36 +++ .../onvoid/webrtc/client/LocalAudioImpl.java | 99 +++++++- .../onvoid/webrtc/client/LocalMediaImpl.java | 4 +- .../java/dev/onvoid/webrtc/client/WebRtc.java | 6 +- .../webrtc/media/audio/AudioProcessing.java | 19 ++ .../onvoid/webrtc/client/LocalAudioTests.java | 163 +++++++++++++ .../webrtc/media/audio/CapturedAudioTest.java | 224 ++++++++++++++++++ 17 files changed, 907 insertions(+), 10 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index d9ca6ea8f..a01d58955 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -522,6 +522,15 @@ A handler that falls behind gets the audio of the last 5 seconds; the oldest aud track.asAudio().onAudio(Duration.ofMillis(200), chunk -> recognizer.feed(chunk.samples())); ``` +A local audio track has `onAudio` too, with the same buffers, for what this side sends, such as for a voice activity detector or a level meter: + +```java +LocalAudio microphone = rtc.media().microphone(); +microphone.onAudio(Duration.ofMillis(200), chunk -> detector.feed(chunk)); +``` + +Pushed audio arrives as pushed. A microphone's audio arrives after audio processing, with echo cancellation, noise suppression and gain control applied as configured, which is what the remote peer hears: the remote peer's voice from the speakers does not count as this side speaking. It flows while WebRTC records, which it does while a peer sends a microphone track. Every microphone track of a runtime has the same audio, as WebRTC records one microphone for all of them. + ### Receiving Without Sending A viewer that sends nothing asks for what it wants to receive, so that the offer includes it: diff --git a/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h b/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h index 8e8644ed3..9a0fefc34 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h +++ b/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h @@ -7,6 +7,14 @@ #ifdef __cplusplus extern "C" { #endif + /* + * Class: dev_onvoid_webrtc_media_audio_AudioProcessing + * Method: setCapturedAudioSink + * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_setCapturedAudioSink + (JNIEnv*, jobject, jobject); + /* * Class: dev_onvoid_webrtc_media_audio_AudioProcessing * Method: applyConfig diff --git a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h index 6490f3dfc..93f485818 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h @@ -23,6 +23,14 @@ extern "C" { JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_createAudioTrackInternal (JNIEnv *, jobject, jstring, jobject); + /* + * Class: dev_onvoid_webrtc_PeerConnectionFactory + * Method: setCapturedAudioSink + * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_setCapturedAudioSink + (JNIEnv *, jobject, jobject); + /* * Class: dev_onvoid_webrtc_PeerConnectionFactory * Method: setDeviceCaptureEnabled diff --git a/webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h b/webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h new file mode 100644 index 000000000..cd0f49da2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h @@ -0,0 +1,63 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_AUDIO_CAPTURE_TAP_H_ +#define JNI_WEBRTC_MEDIA_AUDIO_CAPTURE_TAP_H_ + +#include "api/audio/audio_processing.h" +#include "api/AudioTrackSink.h" + +#include +#include +#include +#include + +namespace jni +{ + // Hands the capture signal of an AudioProcessing, after all of its + // processing, to a sink: what the senders send. The module calls it + // through its capture post-processor, which is put in when the module is + // built, so the tap does nothing until a sink is set. + // + // The tap is shared: by the post-processor, for as long as the module + // lives, and by the Java objects that set its sink, the module's and the + // factory's, for as long as they live. None of them outlives it. + class AudioCaptureTap + { + public: + // The capture post-processor to build an AudioProcessing with. + static std::unique_ptr CreateProcessing(std::shared_ptr tap); + + // Replaces the sink, or stops the delivery with nullptr. Once this + // returns, the sink replaced is no longer called. Must not be + // called from within the sink. + void SetSink(std::unique_ptr sink); + + // Called on the capture thread. + void Process(const webrtc::AudioBuffer & audio); + + private: + std::mutex mutex; + + // Guarded by mutex. + std::unique_ptr sink; + + // Guarded by mutex. + std::vector samples; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp index 3f7547b1c..7ad260dd4 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp @@ -25,6 +25,8 @@ #include "JavaUtils.h" #include "WebRTCContext.h" +#include "api/AudioTrackSink.h" +#include "media/audio/AudioCaptureTap.h" #include "media/audio/AudioProcessing.h" #include "media/audio/AudioProcessingConfig.h" #include "media/audio/AudioProcessingStreamConfig.h" @@ -38,6 +40,25 @@ #include #include +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_setCapturedAudioSink +(JNIEnv * env, jobject caller, jobject jSink) +{ + try { + webrtc::AudioProcessing * apm = GetHandle(env, caller); + CHECK_HANDLE(apm); + + auto tap = GetHandle>(env, caller, "captureTapHandle"); + CHECK_HANDLE(tap); + + (*tap)->SetSink(jSink == nullptr + ? nullptr + : std::make_unique(env, jni::JavaGlobalRef(env, jSink), false)); + } + catch (...) { + ThrowCxxJavaException(env); + } +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_applyConfig (JNIEnv * env, jobject caller, jobject config) { @@ -168,6 +189,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_dispos SetHandle(env, caller, nullptr); apm = nullptr; + + // The module, if a factory still uses it, keeps a share of the tap. + delete GetHandle>(env, caller, "captureTapHandle"); + + SetHandle(env, caller, "captureTapHandle", nullptr); } JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_initialize @@ -175,7 +201,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_initia { jni::WebRTCContext * context = static_cast(javaContext); - webrtc::scoped_refptr apm = webrtc::BuiltinAudioProcessingBuilder().Build(context->webrtcEnv); + auto tap = std::make_shared(); + + webrtc::scoped_refptr apm = webrtc::BuiltinAudioProcessingBuilder() + .SetCapturePostProcessing(jni::AudioCaptureTap::CreateProcessing(tap)) + .Build(context->webrtcEnv); if (!apm) { env->Throw(jni::JavaError(env, "Create AudioProcessing failed")); @@ -183,6 +213,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_initia } SetHandle(env, caller, apm.release()); + SetHandle(env, caller, "captureTapHandle", new std::shared_ptr(std::move(tap))); } JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_updateStats diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index 19c3c6fd1..5961a5480 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -33,6 +33,8 @@ #include "WebRTCContext.h" #include "api/audio/builtin_audio_processing_builder.h" +#include "api/AudioTrackSink.h" +#include "media/audio/AudioCaptureTap.h" #include "api/audio/create_audio_device_module.h" #include "api/create_modular_peer_connection_factory.h" #include "api/enable_media.h" @@ -157,9 +159,29 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize dependencies.adm = proxy; dependencies.audio_encoder_factory = audioEncoderFactory; dependencies.audio_decoder_factory = audioDecoderFactory; - dependencies.audio_processing_builder = (processing != nullptr) - ? webrtc::CustomAudioProcessing(webrtc::scoped_refptr(processing)) - : std::make_unique(); + // The tap of the module that processes the captured audio: that of the + // application's module, or of the one built here. + std::shared_ptr tap; + + if (processing != nullptr) { + auto moduleTap = GetHandle>(env, audioProcessing, "captureTapHandle"); + + if (moduleTap == nullptr) { + throw jni::Exception("The AudioProcessing was disposed"); + } + + tap = *moduleTap; + dependencies.audio_processing_builder = webrtc::CustomAudioProcessing( + webrtc::scoped_refptr(processing)); + } + else { + tap = std::make_shared(); + + auto builder = std::make_unique(); + builder->SetCapturePostProcessing(jni::AudioCaptureTap::CreateProcessing(tap)); + + dependencies.audio_processing_builder = std::move(builder); + } dependencies.video_encoder_factory = std::move(videoEncoderFactory); dependencies.video_decoder_factory = std::move(videoDecoderFactory); @@ -172,6 +194,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize SetHandle(env, caller, "networkThreadHandle", networkThread.release()); SetHandle(env, caller, "signalingThreadHandle", signalingThread.release()); SetHandle(env, caller, "audioModuleHandle", proxy.release()); + SetHandle(env, caller, "captureTapHandle", new std::shared_ptr(std::move(tap))); } else { throw jni::Exception("Create PeerConnectionFactory failed"); @@ -222,6 +245,30 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_dispose SetHandle(env, caller, "audioModuleHandle", nullptr); } + + // The module, if anything still holds it, keeps a share of the tap. + delete GetHandle>(env, caller, "captureTapHandle"); + + SetHandle(env, caller, "captureTapHandle", nullptr); + } + catch (...) { + ThrowCxxJavaException(env); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_setCapturedAudioSink +(JNIEnv * env, jobject caller, jobject jSink) +{ + try { + webrtc::PeerConnectionFactoryInterface * factory = GetHandle(env, caller); + CHECK_HANDLE(factory); + + auto tap = GetHandle>(env, caller, "captureTapHandle"); + CHECK_HANDLE(tap); + + (*tap)->SetSink(jSink == nullptr + ? nullptr + : std::make_unique(env, jni::JavaGlobalRef(env, jSink), false)); } catch (...) { ThrowCxxJavaException(env); diff --git a/webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp b/webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp new file mode 100644 index 000000000..86bad01c2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/AudioCaptureTap.h" + +#include "common_audio/include/audio_util.h" +#include "modules/audio_processing/audio_buffer.h" + +#include + +namespace jni +{ + namespace + { + class CaptureTapProcessing : public webrtc::CustomProcessing + { + public: + explicit CaptureTapProcessing(std::shared_ptr tap) : + tap(std::move(tap)) + { + } + + void Initialize(int sampleRateHz, int numChannels) override + { + // The tap reads the format from each buffer. + } + + void Process(webrtc::AudioBuffer * audio) override + { + tap->Process(*audio); + } + + std::string ToString() const override + { + return "AudioCaptureTap"; + } + + private: + const std::shared_ptr tap; + }; + } + + std::unique_ptr AudioCaptureTap::CreateProcessing(std::shared_ptr tap) + { + return std::make_unique(std::move(tap)); + } + + void AudioCaptureTap::SetSink(std::unique_ptr next) + { + std::unique_ptr replaced; + + { + // Waits for a delivery in progress, which holds the lock. + std::lock_guard lock(mutex); + + replaced = std::move(sink); + sink = std::move(next); + } + + // Deleted outside the lock: it releases a Java reference. + replaced.reset(); + } + + void AudioCaptureTap::Process(const webrtc::AudioBuffer & audio) + { + std::lock_guard lock(mutex); + + if (!sink) { + return; + } + + const size_t channels = audio.num_channels(); + const size_t frames = audio.num_frames(); + const float * const * data = audio.channels_const(); + + samples.resize(channels * frames); + + // The buffer holds one channel after another, as floats in the range + // of 16-bit samples; a sink takes them interleaved. + for (size_t channel = 0; channel < channels; channel++) { + for (size_t frame = 0; frame < frames; frame++) { + samples[frame * channels + channel] = webrtc::FloatS16ToS16(data[channel][frame]); + } + } + + // AudioProcessing works on 10 ms at a time. + sink->OnData(samples.data(), 16, static_cast(frames * 100), channels, frames); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index a392f8b37..8d7b47ad0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -23,6 +23,7 @@ import dev.onvoid.webrtc.media.audio.AudioDeviceModuleBase; import dev.onvoid.webrtc.media.audio.AudioOptions; import dev.onvoid.webrtc.media.audio.AudioProcessing; +import dev.onvoid.webrtc.media.audio.AudioTrackSink; import dev.onvoid.webrtc.media.audio.AudioTrackSource; import dev.onvoid.webrtc.media.audio.AudioTrack; import dev.onvoid.webrtc.media.audio.CustomAudioSource; @@ -102,6 +103,9 @@ private enum AudioInputMode { @SuppressWarnings("unused") private long audioModuleHandle; + @SuppressWarnings("unused") + private long captureTapHandle; + /** Guards {@link #audioInputMode}. */ private final Object audioInputLock = new Object(); @@ -407,6 +411,26 @@ public RTCPeerConnection createPeerConnection(RTCConfiguration config, */ public native RTCRtpCapabilities getRtpSenderCapabilities(MediaType type); + /** + * Sets the sink that gets the microphone audio after audio processing, + * with echo cancellation, noise suppression and gain control applied as + * configured: the audio the senders send. It comes from the {@link + * AudioProcessing} the factory was built with, or else from the one the + * factory built itself, and is the same sink as that module's {@link + * AudioProcessing#setCapturedAudioSink}. + *

+ * Audio arrives while the audio device module records, which WebRTC starts + * once a peer connection sends a track of {@link #createAudioSource}. Each + * call holds 10 ms of 16-bit PCM with interleaved channels, at the rate + * the module processes at. The sink runs on WebRTC's capture thread, so it + * must return promptly, and it must not set the sink itself. Audio pushed + * through a {@link CustomAudioSource} does not pass audio processing and + * does not arrive. + * + * @param sink The sink, or {@code null} to stop the delivery. + */ + public native void setCapturedAudioSink(AudioTrackSink sink); + @Override public native void dispose(); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java new file mode 100644 index 000000000..cbbed3bf6 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.media.audio.AudioTrackSink; + +/** + * Hands the microphone audio of a runtime, after audio processing, to the + * audio subscriptions of its microphone tracks. The factory's audio + * processing calls it on WebRTC's capture thread, and only while one is + * subscribed, so that the capture thread does not call into Java for nothing. + */ +final class CapturedAudio implements AudioTrackSink { + + private final PeerConnectionFactory factory; + + /** Changed under this object's lock, read on the capture thread. */ + private final List sinks = new CopyOnWriteArrayList<>(); + + + CapturedAudio(PeerConnectionFactory factory) { + this.factory = factory; + } + + synchronized void add(AudioTrackSink sink) { + if (sinks.isEmpty()) { + factory.setCapturedAudioSink(this); + } + + sinks.add(sink); + } + + synchronized void remove(AudioTrackSink sink) { + if (sinks.remove(sink) && sinks.isEmpty()) { + factory.setCapturedAudioSink(null); + } + } + + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { + // Each sink copies what it keeps. + for (AudioTrackSink sink : sinks) { + sink.onData(data, bitsPerSample, sampleRate, channels, frames); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java index fdb55d3da..fc9426bd8 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java @@ -82,7 +82,7 @@ final class CustomAudioImpl extends LocalAudioImpl implements CustomAudio { CustomAudioImpl(LocalMediaImpl media, AudioTrack track, CustomAudioSource source) { - super(media, track, source::dispose); + super(media, track, source::dispose, false); this.source = source; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java index fda582a2a..cb37a98b2 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java @@ -16,9 +16,45 @@ package dev.onvoid.webrtc.client; +import java.time.Duration; +import java.util.function.Consumer; + /** * An audio track this side sends. */ public interface LocalAudio extends LocalTrack { + /** + * Calls the handler with the audio this track sends, in order, such as + * for a voice activity detector or a level meter. Pushed audio arrives as + * pushed. A microphone's audio arrives after audio processing, with echo + * cancellation, noise suppression and gain control applied as configured, + * so that the remote peer's voice from the speakers does not count as + * this side speaking. It flows while WebRTC records, which it does while + * a peer sends a microphone track, and every microphone track of a + * runtime has the same, as WebRTC records one microphone for all of them. + *

+ * A handler that falls behind gets the audio of the last {@link + * WebRtc.Builder#audioBuffer(Duration) audio buffer} of the runtime; the + * oldest audio beyond that is dropped. + * + * @param handler called on a thread of this subscription. + * + * @return the subscription that stops the audio. + */ + Subscription onAudio(Consumer handler); + + /** + * Like {@link #onAudio(Consumer)}, with a buffer of its own: a handler + * that falls behind gets the audio of the last {@code buffer}, and the + * oldest audio beyond that is dropped. A buffer shorter than one chunk + * keeps only the newest chunk. + * + * @param buffer the most audio that waits for the handler. + * @param handler called on a thread of this subscription. + * + * @return the subscription that stops the audio. + */ + Subscription onAudio(Duration buffer, Consumer handler); + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java index 3d6287181..720d431fd 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java @@ -16,15 +16,110 @@ package dev.onvoid.webrtc.client; +import static java.util.Objects.requireNonNull; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.internal.Timeouts; import dev.onvoid.webrtc.media.audio.AudioTrack; /** - * A local audio track, from the microphone or pushed audio. + * A local audio track, from the microphone or pushed audio. Each audio + * subscription has a thread of its own. Pushed audio reaches it as a sink of + * the native track. A microphone's source hands no audio to sinks, as WebRTC + * passes captured audio straight to the senders, so a microphone track takes + * it from the runtime's audio processing instead. */ class LocalAudioImpl extends LocalTrackImpl implements LocalAudio { - LocalAudioImpl(LocalMediaImpl media, AudioTrack track, Runnable releaseSource) { + private final boolean microphone; + + // Guarded by this. + + private final List dispatchers = new ArrayList<>(); + + + /** + * @param microphone true for a track of captured audio, false for one + * whose source hands its audio to sinks. + */ + LocalAudioImpl(LocalMediaImpl media, AudioTrack track, Runnable releaseSource, boolean microphone) { super(media, track, MediaKind.AUDIO, releaseSource); + + this.microphone = microphone; } + @Override + public Subscription onAudio(Consumer handler) { + return onAudio(rtc.audioBuffer, handler); + } + + @Override + public Subscription onAudio(Duration buffer, Consumer handler) { + Timeouts.requirePositive(buffer, "buffer"); + requireNonNull(handler, "handler"); + + SerialExecutor executor = new SerialExecutor(rtc.framePool, rtc::reportError, false, null); + AudioDispatcher dispatcher = new AudioDispatcher(handler, buffer, executor, rtc.audioWork, + rtc::reportError); + + synchronized (this) { + if (!isLive()) { + return () -> { }; + } + + attach(dispatcher); + dispatchers.add(dispatcher); + } + + return () -> { + dispatcher.close(); + + synchronized (LocalAudioImpl.this) { + if (dispatchers.remove(dispatcher) && isLive()) { + detach(dispatcher); + } + } + }; + } + + @Override + void beforeRelease() { + List attached; + + synchronized (this) { + attached = new ArrayList<>(dispatchers); + dispatchers.clear(); + } + + for (AudioDispatcher dispatcher : attached) { + detach(dispatcher); + dispatcher.close(); + } + } + + private void attach(AudioDispatcher dispatcher) { + if (microphone) { + rtc.capturedAudio.add(dispatcher); + } + else { + audioTrack().addSink(dispatcher); + } + } + + private void detach(AudioDispatcher dispatcher) { + if (microphone) { + rtc.capturedAudio.remove(dispatcher); + } + else { + audioTrack().removeSink(dispatcher); + } + } + + private AudioTrack audioTrack() { + return (AudioTrack) nativeTrack(); + } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java index 7e5560f7c..eea46fa33 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java @@ -162,7 +162,7 @@ public LocalAudio microphone(AudioOptions options) { try { AudioTrack track = rtc.factory.createAudioTrack(newId("microphone"), source); - return register(new LocalAudioImpl(this, track, source::dispose)); + return register(new LocalAudioImpl(this, track, source::dispose, true)); } catch (RuntimeException e) { source.dispose(); @@ -254,7 +254,7 @@ public LocalAudio audioTrack(String label, AudioTrackSource source) { AudioTrack track = rtc.factory.createAudioTrack(label, source); // The source is the caller's, and outlives the track. - return register(new LocalAudioImpl(this, track, () -> { })); + return register(new LocalAudioImpl(this, track, () -> { }, !pushed)); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java index a7b2c32b1..e64b7d08f 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java @@ -132,7 +132,10 @@ public final class WebRtc implements AutoCloseable { private final AudioDeviceModuleBase audioModule; - private final AudioProcessing audioProcessing; + /** Created when the runtime was built with an audio processing config. */ + final AudioProcessing audioProcessing; + + final CapturedAudio capturedAudio; private final Duration closeTimeout; @@ -260,6 +263,7 @@ private WebRtc(Builder builder) { this.audioModule = module; this.audioProcessing = processing; this.factory = created; + this.capturedAudio = new CapturedAudio(created); this.media = new LocalMediaImpl(this, module); } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java index 91a0eb6fe..1df8d67ef 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java @@ -41,6 +41,9 @@ public class AudioProcessing extends DisposableNativeObject { /** Cached statistics object to avoid recreation. */ private final AudioProcessingStats stats = new AudioProcessingStats(); + @SuppressWarnings("unused") + private long captureTapHandle; + /** * Creates a new instance of {@code AudioProcessing}. Make sure to call @@ -156,6 +159,22 @@ public native int processReverseStream(byte[] src, AudioProcessingStreamConfig inputConfig, AudioProcessingStreamConfig outputConfig, byte[] dest); + /** + * Sets the sink that gets the capture stream after all processing, as + * {@link #processStream} puts it out, or, for a module a {@link + * dev.onvoid.webrtc.PeerConnectionFactory} was built with, the microphone + * audio its senders send. Each call holds 10 ms of 16-bit PCM with + * interleaved channels, at the rate and channel count the module processes + * at, which may differ from the format {@code processStream} puts out. + *

+ * The sink runs on the thread that processes the capture stream, WebRTC's + * capture thread for a factory, so it must return promptly, and it must + * not set the sink itself. + * + * @param sink The sink, or {@code null} to stop the delivery. + */ + public native void setCapturedAudioSink(AudioTrackSink sink); + @Override public native void dispose(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java new file mode 100644 index 000000000..e342ed178 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java @@ -0,0 +1,163 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; + +import dev.onvoid.webrtc.media.audio.AudioOptions; +import dev.onvoid.webrtc.media.audio.AudioProcessingConfig; +import dev.onvoid.webrtc.media.audio.AudioProcessingStreamConfig; +import dev.onvoid.webrtc.media.audio.AudioTrackSource; + +import org.junit.jupiter.api.Test; + +/** + * The audio a local track sends: pushed audio as pushed, a microphone's after + * the runtime's audio processing. + */ +class LocalAudioTests extends ClientTestBase { + + private static final PcmFormat FORMAT = PcmFormat.of(48000, 1); + + + @Test + void pushedAudioArrives() throws Exception { + CustomAudio audio = rtc.media().customAudio("audio"); + CompletableFuture received = new CompletableFuture<>(); + + audio.onAudio(received::complete); + audio.push(constant(4800, 1000), FORMAT); + + AudioChunk chunk = await(received); + + assertEquals(48000, chunk.sampleRate()); + assertEquals(1, chunk.channels()); + assertSamples(1000, chunk); + } + + @Test + void microphoneAudioAfterProcessing() throws Exception { + // A fixed gain makes what was processed differ from what went in. + AudioProcessingConfig config = new AudioProcessingConfig(); + config.gainControllerDigital.enabled = true; + config.gainControllerDigital.fixedDigital.gainDb = 6; + + WebRtc runtime = WebRtc.builder() + .audio(AudioMode.HEADLESS) + .audioProcessing(config) + .errorHandler(errors::add) + .build(); + + try { + LocalAudio microphone = microphone(runtime); + CompletableFuture first = new CompletableFuture<>(); + CompletableFuture second = new CompletableFuture<>(); + + Subscription one = microphone.onAudio(first::complete); + microphone.onAudio(second::complete); + + // Stands in for WebRTC, which hands the microphone's audio to the + // same audio processing before it sends it. + byte[] input = constant(480, 1000); + byte[] output = new byte[input.length]; + AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); + + assertEquals(0, runtime.audioProcessing.processStream(input, format, format, output)); + + AudioChunk chunk = await(first); + + assertEquals(48000, chunk.sampleRate()); + assertEquals(480, chunk.frames()); + assertArrayEquals(output, chunk.samples()); + assertFalse(Arrays.equals(input, chunk.samples())); + assertArrayEquals(output, await(second).samples()); + + // The other subscription keeps the audio coming. + one.close(); + + CompletableFuture third = new CompletableFuture<>(); + microphone.onAudio(third::complete); + runtime.audioProcessing.processStream(input, format, format, output); + + assertArrayEquals(output, await(third).samples()); + + // Closing the track ends its subscriptions. + await(microphone.closeAsync()); + + Subscription late = microphone.onAudio(chunkAfter -> { }); + late.close(); + } + finally { + runtime.close(); + } + } + + @Test + void microphoneWithFactorysOwnProcessing() throws Exception { + // Without a config, the factory builds its audio processing itself; + // subscribing and closing needs nothing of the application. + LocalAudio microphone = microphone(rtc); + + Subscription audio = microphone.onAudio(chunk -> { }); + audio.close(); + microphone.onAudio(chunk -> { }); + + await(microphone.closeAsync()); + + assertTrue(microphone.isEnded()); + } + + /** + * A microphone track of a headless runtime, which records nothing, so + * that the test hands its audio processing the audio. + */ + private static LocalAudio microphone(WebRtc runtime) { + LocalMediaImpl media = (LocalMediaImpl) runtime.media(); + AudioTrackSource source = runtime.factory.createAudioSource(new AudioOptions()); + + return media.register(new LocalAudioImpl(media, + runtime.factory.createAudioTrack("microphone", source), source::dispose, true)); + } + + /** Mono 16-bit PCM with every sample set to the value. */ + private static byte[] constant(int frames, int value) { + ByteBuffer pcm = ByteBuffer.allocate(frames * 2).order(ByteOrder.LITTLE_ENDIAN); + + for (int i = 0; i < frames; i++) { + pcm.putShort((short) value); + } + + return pcm.array(); + } + + private static void assertSamples(int value, AudioChunk chunk) { + ByteBuffer samples = ByteBuffer.wrap(chunk.samples()).order(ByteOrder.LITTLE_ENDIAN); + + while (samples.hasRemaining()) { + assertEquals(value, samples.getShort()); + } + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java new file mode 100644 index 000000000..b263cfef6 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java @@ -0,0 +1,224 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import dev.onvoid.webrtc.PeerConnectionFactory; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * The capture stream of an {@link AudioProcessing} after all processing, + * taken by a sink, which is what a factory's senders send. + */ +class CapturedAudioTest { + + private final List captured = new CopyOnWriteArrayList<>(); + + private final AudioTrackSink sink = (data, bitsPerSample, sampleRate, channels, frames) -> + captured.add(new Captured(data, bitsPerSample, sampleRate, channels, frames)); + + private AudioProcessing processing; + + + @BeforeEach + void createProcessing() { + processing = new AudioProcessing(); + } + + @AfterEach + void disposeProcessing() { + processing.dispose(); + } + + @Test + void sinkGetsProcessedStream() { + // A fixed gain makes what was processed differ from what went in. + AudioProcessingConfig config = new AudioProcessingConfig(); + config.gainControllerDigital.enabled = true; + config.gainControllerDigital.fixedDigital.gainDb = 6; + processing.applyConfig(config); + processing.setCapturedAudioSink(sink); + + AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); + byte[] input = tone(48000, 1); + byte[] output = new byte[input.length]; + + assertEquals(0, processing.processStream(input, format, format, output)); + + assertEquals(1, captured.size()); + + Captured chunk = captured.get(0); + + assertEquals(16, chunk.bitsPerSample); + assertEquals(48000, chunk.sampleRate); + assertEquals(1, chunk.channels); + assertEquals(480, chunk.frames); + // Exactly what the stream puts out, after the gain. + assertArrayEquals(output, chunk.data); + assertFalse(Arrays.equals(input, chunk.data)); + } + + @Test + void sinkGetsProcessingRate() { + processing.setCapturedAudioSink(sink); + + AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(16000, 1); + byte[] input = tone(16000, 1); + + processing.processStream(input, format, format, new byte[input.length]); + + Captured chunk = captured.get(0); + + assertEquals(16000, chunk.sampleRate); + assertEquals(160, chunk.frames); + assertEquals(chunk.frames * chunk.channels * 2, chunk.data.length); + } + + @Test + void sinkRemoved() { + AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); + byte[] input = tone(48000, 1); + + // None set yet. + processing.processStream(input, format, format, new byte[input.length]); + + processing.setCapturedAudioSink(sink); + processing.processStream(input, format, format, new byte[input.length]); + + processing.setCapturedAudioSink(null); + processing.processStream(input, format, format, new byte[input.length]); + + assertEquals(1, captured.size()); + } + + @Test + void factoryTakesSinkOfItsProcessing() { + HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); + PeerConnectionFactory factory = new PeerConnectionFactory(module, processing); + + try { + factory.setCapturedAudioSink(sink); + + // What the factory's audio processing processes, as it would the + // microphone's audio. + AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); + byte[] input = tone(48000, 1); + processing.processStream(input, format, format, new byte[input.length]); + + assertEquals(1, captured.size()); + + factory.setCapturedAudioSink(null); + processing.processStream(input, format, format, new byte[input.length]); + + assertEquals(1, captured.size()); + } + finally { + factory.dispose(); + module.dispose(); + } + } + + @Test + void factoryWithOwnProcessing() { + HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); + PeerConnectionFactory factory = new PeerConnectionFactory(module); + + try { + factory.setCapturedAudioSink(sink); + factory.setCapturedAudioSink(null); + factory.setCapturedAudioSink(sink); + } + finally { + // Releases the sink still set along with the module. + factory.dispose(); + module.dispose(); + } + + assertThrows(NullPointerException.class, () -> factory.setCapturedAudioSink(sink)); + } + + @Test + void processingOutlivesItsObject() { + // The factory keeps the module, and with it the sink, after the + // application disposed of its object. + AudioProcessing shared = new AudioProcessing(); + HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); + PeerConnectionFactory factory = new PeerConnectionFactory(module, shared); + + try { + shared.dispose(); + + factory.setCapturedAudioSink(sink); + factory.setCapturedAudioSink(null); + + assertThrows(NullPointerException.class, () -> shared.setCapturedAudioSink(sink)); + } + finally { + factory.dispose(); + module.dispose(); + } + } + + /** 10 ms of a 440 Hz tone. */ + private static byte[] tone(int sampleRate, int channels) { + int frames = sampleRate / 100; + byte[] pcm = new byte[frames * channels * 2]; + + for (int i = 0; i < frames; i++) { + short sample = (short) (Math.sin(2 * Math.PI * 440 * i / sampleRate) * 4000); + + for (int channel = 0; channel < channels; channel++) { + int offset = (i * channels + channel) * 2; + pcm[offset] = (byte) sample; + pcm[offset + 1] = (byte) (sample >> 8); + } + } + + return pcm; + } + + + + private static final class Captured { + + final byte[] data; + final int bitsPerSample; + final int sampleRate; + final int channels; + final int frames; + + + Captured(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { + this.data = data.clone(); + this.bitsPerSample = bitsPerSample; + this.sampleRate = sampleRate; + this.channels = channels; + this.frames = frames; + } + } +} From 09cc97f1087a3f16a96a3ad7cbde342481f71ed3 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 29 Oct 2026 20:06:14 +0100 Subject: [PATCH 3/4] docs: detect speech at the microphone with LocalAudio.onAudio The VAD guide, VoiceActivityExample and the audio processing guide show the processed microphone audio of the client API and of PeerConnectionFactory.setCapturedAudioSink. --- docs/guide/audio/audio-processing.md | 15 +++++++++++++ docs/tools/audio/voice-activity-detector.md | 21 ++++++++++++++++--- .../webrtc/examples/VoiceActivityExample.java | 9 +++++--- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/docs/guide/audio/audio-processing.md b/docs/guide/audio/audio-processing.md index 92d5c29b4..c6358ad7e 100644 --- a/docs/guide/audio/audio-processing.md +++ b/docs/guide/audio/audio-processing.md @@ -296,6 +296,21 @@ audioProcessing.processStream( highRateBuffer, highRateConfig, lowRateConfig, lowRateDestBuffer); ``` +## Processed Microphone Audio + +WebRTC passes the microphone's audio from the audio device module straight to the senders, so a sink on a microphone track gets nothing. The audio as it is sent, after echo cancellation, noise suppression and gain control, goes to the sink of the factory's audio processing instead, such as for a voice activity detector or a level meter: + +```java +factory.setCapturedAudioSink((data, bitsPerSample, sampleRate, channels, frames) -> { + // 10 ms of 16-bit PCM, on WebRTC's capture thread: hand it on and return. +}); + +// Later, to stop it. +factory.setCapturedAudioSink(null); +``` + +It works with the `AudioProcessing` the factory was built with as well as with the one it builds itself; for a module of its own, `AudioProcessing.setCapturedAudioSink` sets the same sink and also gets what `processStream` puts out. Audio arrives while the audio device module records, which WebRTC starts once a peer connection sends a microphone track, at the rate and channel count the module processes at. The sink must not set the sink itself. The [Client API](/guide/client-api#audio) offers the same through `LocalAudio.onAudio`. + ## Statistics The `AudioProcessing` class provides statistics about the audio processing performance: diff --git a/docs/tools/audio/voice-activity-detector.md b/docs/tools/audio/voice-activity-detector.md index 4e9aa8c17..734bcb7b4 100644 --- a/docs/tools/audio/voice-activity-detector.md +++ b/docs/tools/audio/voice-activity-detector.md @@ -125,8 +125,21 @@ The examples module includes `VoiceActivityExample`, an adapter using converts mono or stereo input to 16 kHz mono with `AudioConverter`, and reports each boolean decision. It also supports 44.1 kHz input. -With the [Client API](/guide/client-api), it takes the `AudioChunk`s of a remote -track's `onAudio` subscription: +With the [Client API](/guide/client-api), it takes the `AudioChunk`s of an +`onAudio` subscription. To detect speech at the microphone, subscribe to the +microphone track. Its audio arrives after echo cancellation and noise +suppression, so the remote peer's voice from the speakers does not count as +this side speaking: + +```java +VoiceActivityExample vad = new VoiceActivityExample(speech -> { + // Enqueue the decision for your application/UI; return promptly. +}); +Subscription audio = microphone.onAudio(Duration.ofMillis(200), vad); +``` + +The microphone's audio flows while a peer sends the track. For what the remote +peer sends, subscribe to its track: ```java import dev.onvoid.webrtc.client.Subscription; @@ -167,7 +180,9 @@ finally { A microphone track does not: WebRTC passes captured audio from the audio device module straight to the senders, and the source of the track hands nothing to -sinks. An [`AudioRecorder`](/tools/audio/audio-recorder) captures a microphone +sinks. Its audio after processing goes to the sink set with +`PeerConnectionFactory.setCapturedAudioSink`, which is what `LocalAudio.onAudio` +uses. An [`AudioRecorder`](/tools/audio/audio-recorder) captures a microphone on its own, apart from any peer connection. The adapter invokes the consumer on the thread that delivers the audio: a thread diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java index 53b4c6bb9..5cc307f20 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java @@ -22,6 +22,7 @@ import java.util.Objects; import dev.onvoid.webrtc.client.AudioChunk; +import dev.onvoid.webrtc.client.LocalAudio; import dev.onvoid.webrtc.client.RemoteAudio; import dev.onvoid.webrtc.client.Subscription; import dev.onvoid.webrtc.media.audio.AudioConverter; @@ -29,9 +30,11 @@ import dev.onvoid.webrtc.media.audio.StandaloneVoiceActivityDetector; /** - * With the client API, subscribe it to a remote track with - * {@link RemoteAudio#onAudio(Consumer)}, then close the {@link Subscription} - * before close(). An AudioTrack of the native API takes it with addSink(), + * With the client API, subscribe it to the microphone with + * {@link LocalAudio#onAudio(Consumer)}, whose audio arrives after echo + * cancellation, or to a remote track with {@link RemoteAudio#onAudio(Consumer)}, + * then close the {@link Subscription} before close(). An AudioTrack of the + * native API takes it with addSink(), * then removeSink() before close(), if its source hands audio to sinks: that * of a remote track or a CustomAudioSource does, a microphone's does not. * Buffers arbitrary chunk lengths into 10 ms frames and converts interleaved From 4902c466057ac8c84ca8ff650b88b55303100188 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sat, 31 Oct 2026 03:01:59 +0100 Subject: [PATCH 4/4] feat: detect speech natively for local audio tracks Replaces LocalAudio.onAudio with onVoiceActivity: the native VAD runs in the audio processing tap (microphone) or in a track sink (pushed audio), and only speech start and end reach Java. VoiceActivityConfig sets threshold and hangover. --- docs/guide/audio/audio-processing.md | 16 +- docs/guide/client-api.md | 8 +- docs/tools/audio/voice-activity-detector.md | 73 +++- .../webrtc/examples/VoiceActivityExample.java | 9 +- .../main/cpp/include/JNI_AudioProcessing.h | 8 - .../src/main/cpp/include/JNI_AudioTrack.h | 16 + .../cpp/include/JNI_PeerConnectionFactory.h | 20 +- .../cpp/include/media/audio/AudioCaptureTap.h | 31 +- .../media/audio/VoiceActivityListener.h | 69 ++++ .../include/media/audio/VoiceActivitySink.h | 49 +++ .../cpp/include/media/audio/VoiceDetector.h | 68 ++++ .../src/main/cpp/src/JNI_AudioProcessing.cpp | 20 -- .../src/main/cpp/src/JNI_AudioTrack.cpp | 37 ++ .../cpp/src/JNI_PeerConnectionFactory.cpp | 31 +- .../cpp/src/media/audio/AudioCaptureTap.cpp | 55 ++- .../src/media/audio/VoiceActivityListener.cpp | 72 ++++ .../cpp/src/media/audio/VoiceActivitySink.cpp | 40 +++ .../cpp/src/media/audio/VoiceDetector.cpp | 96 +++++ .../onvoid/webrtc/PeerConnectionFactory.java | 94 ++++- .../onvoid/webrtc/client/CapturedAudio.java | 64 ---- .../dev/onvoid/webrtc/client/LocalAudio.java | 47 ++- .../onvoid/webrtc/client/LocalAudioImpl.java | 101 ++++-- .../java/dev/onvoid/webrtc/client/WebRtc.java | 3 - .../webrtc/media/audio/AudioProcessing.java | 16 - .../onvoid/webrtc/media/audio/AudioTrack.java | 56 +++ .../media/audio/VoiceActivityConfig.java | 97 +++++ .../media/audio/VoiceActivityListener.java | 40 +++ .../onvoid/webrtc/client/LocalAudioTests.java | 140 ++++---- .../webrtc/media/audio/CapturedAudioTest.java | 224 ------------ .../webrtc/media/audio/SpeechFixture.java | 100 ++++++ .../audio/VoiceActivityListenerTest.java | 336 ++++++++++++++++++ 31 files changed, 1485 insertions(+), 551 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h create mode 100644 webrtc-jni/src/main/cpp/include/media/audio/VoiceActivitySink.h create mode 100644 webrtc-jni/src/main/cpp/include/media/audio/VoiceDetector.h create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/VoiceActivitySink.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/VoiceDetector.cpp delete mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityConfig.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityListener.java delete mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SpeechFixture.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java diff --git a/docs/guide/audio/audio-processing.md b/docs/guide/audio/audio-processing.md index c6358ad7e..6afd7db3b 100644 --- a/docs/guide/audio/audio-processing.md +++ b/docs/guide/audio/audio-processing.md @@ -296,20 +296,22 @@ audioProcessing.processStream( highRateBuffer, highRateConfig, lowRateConfig, lowRateDestBuffer); ``` -## Processed Microphone Audio +## Speech in the Processed Microphone Audio -WebRTC passes the microphone's audio from the audio device module straight to the senders, so a sink on a microphone track gets nothing. The audio as it is sent, after echo cancellation, noise suppression and gain control, goes to the sink of the factory's audio processing instead, such as for a voice activity detector or a level meter: +WebRTC passes the microphone's audio from the audio device module straight to the senders, so a sink on a microphone track gets nothing. The factory detects speech in that audio as it is sent, after echo cancellation, noise suppression and gain control, in native code, and tells a listener only when speech begins and ends: ```java -factory.setCapturedAudioSink((data, bitsPerSample, sampleRate, channels, frames) -> { - // 10 ms of 16-bit PCM, on WebRTC's capture thread: hand it on and return. -}); +VoiceActivityListener listener = speaking -> { + // On WebRTC's capture thread: hand it on and return. +}; + +factory.addVoiceActivityListener(new VoiceActivityConfig(0.5, Duration.ofMillis(300)), listener); // Later, to stop it. -factory.setCapturedAudioSink(null); +factory.removeVoiceActivityListener(listener); ``` -It works with the `AudioProcessing` the factory was built with as well as with the one it builds itself; for a module of its own, `AudioProcessing.setCapturedAudioSink` sets the same sink and also gets what `processStream` puts out. Audio arrives while the audio device module records, which WebRTC starts once a peer connection sends a microphone track, at the rate and channel count the module processes at. The sink must not set the sink itself. The [Client API](/guide/client-api#audio) offers the same through `LocalAudio.onAudio`. +It works with the `AudioProcessing` the factory was built with as well as with the one it builds itself. All listeners share one detector, and each decides with the threshold and hangover of its own `VoiceActivityConfig`. Audio arrives while the audio device module records, which WebRTC starts once a peer connection sends a microphone track. A listener must not add or remove a listener. Disposing of the factory removes the listeners it still has. The [Client API](/guide/client-api#audio) offers the same through `LocalAudio.onVoiceActivity`. ## Statistics diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 10453ad59..7f9ca0e35 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -522,14 +522,16 @@ A handler that falls behind gets the audio of the last 5 seconds; the oldest aud track.asAudio().onAudio(Duration.ofMillis(200), chunk -> recognizer.feed(chunk.samples())); ``` -A local audio track has `onAudio` too, with the same buffers, for what this side sends, such as for a voice activity detector or a level meter: +A local audio track tells when this side speaks. `onVoiceActivity` calls its handler with `true` when speech begins and `false` when it ends, on the event executor. The audio is analyzed in native code, and only these changes reach Java: ```java LocalAudio microphone = rtc.media().microphone(); -microphone.onAudio(Duration.ofMillis(200), chunk -> detector.feed(chunk)); +Subscription speaking = microphone.onVoiceActivity( + new VoiceActivityConfig(0.6, Duration.ofMillis(400)), + active -> System.out.println(active ? "speaking" : "quiet")); ``` -Pushed audio arrives as pushed. A microphone's audio arrives after audio processing, with echo cancellation, noise suppression and gain control applied as configured, which is what the remote peer hears: the remote peer's voice from the speakers does not count as this side speaking. It flows while WebRTC records, which it does while a peer sends a microphone track. Every microphone track of a runtime has the same audio, as WebRTC records one microphone for all of them. +`VoiceActivityConfig` sets the threshold, the voice probability from 0 to 1 at which audio counts as speech (0.5 by default), and the hangover, how long speech goes on once the audio fell below it (300 ms by default), which bridges the pauses between words. Pushed audio is analyzed as pushed. A microphone's audio is analyzed after audio processing, with echo cancellation, noise suppression and gain control applied as configured, so the remote peer's voice from the speakers does not count as this side speaking. It flows while WebRTC records, which it does while a peer sends a microphone track. Every microphone track of a runtime hears the same, as WebRTC records one microphone for all of them. ### Receiving Without Sending diff --git a/docs/tools/audio/voice-activity-detector.md b/docs/tools/audio/voice-activity-detector.md index 734bcb7b4..bb4206e81 100644 --- a/docs/tools/audio/voice-activity-detector.md +++ b/docs/tools/audio/voice-activity-detector.md @@ -13,6 +13,10 @@ The existing `VoiceActivityDetector.process()` and `getLastVoiceProbability()` methods remain available with the original probability estimator. No migration is required. The standalone detector is an additional class, with its own state. +To learn only when speech begins and ends in a stream, without handing its +audio to Java, add a `VoiceActivityListener`, see +[Speech events in native code](#speech-events-in-native-code). + ## Probability estimator `dev.onvoid.webrtc.media.audio.VoiceActivityDetector` combines the standalone VAD @@ -118,28 +122,59 @@ they receive the same audio. Feeding, configuring, resetting, or disposing one does not update the other. Neither API automatically changes encoding, DTX, mute state, or transmitted audio. -## Audio track integration +## Speech events in native code -The examples module includes `VoiceActivityExample`, an adapter using -`StandaloneVoiceActivityDetector`. It buffers incoming audio into 10 ms frames, -converts mono or stereo input to 16 kHz mono with `AudioConverter`, and reports -each boolean decision. It also supports 44.1 kHz input. +A `VoiceActivityListener` gets the changes only: `true` when speech begins and +`false` when it ends. The probability estimator runs in native code on the +audio as it flows, and a `VoiceActivityConfig` sets when audio counts as speech: -With the [Client API](/guide/client-api), it takes the `AudioChunk`s of an -`onAudio` subscription. To detect speech at the microphone, subscribe to the -microphone track. Its audio arrives after echo cancellation and noise -suppression, so the remote peer's voice from the speakers does not count as -this side speaking: +- the threshold, the probability from which on audio counts as speech, above 0 + and at most 1, 0.5 by default; +- the hangover, how long speech goes on once the audio fell below the + threshold, 300 ms by default, which bridges the pauses between words. + +Audio of any rate and channel count is mixed down to mono and resampled to +16 kHz first. The estimator's initial probability of `1.0` never counts. + +An `AudioTrack` whose source hands its audio to sinks, a remote track or a +track over a `CustomAudioSource`, takes a listener for its audio: ```java -VoiceActivityExample vad = new VoiceActivityExample(speech -> { - // Enqueue the decision for your application/UI; return promptly. +VoiceActivityListener listener = speaking -> { + // On the thread that delivers the audio: enqueue it and return promptly. +}; +track.addVoiceActivityListener(new VoiceActivityConfig(), listener); +try { + // Keep the listener while receiving audio. +} +finally { + track.removeVoiceActivityListener(listener); +} +``` + +A microphone track does not hand its audio to sinks. Its speech comes from the +factory, after echo cancellation and noise suppression, so the remote peer's +voice from the speakers does not count as this side speaking: +`PeerConnectionFactory.addVoiceActivityListener`, see +[Audio Processing](/guide/audio/audio-processing#speech-in-the-processed-microphone-audio). +With the [Client API](/guide/client-api#audio), `LocalAudio.onVoiceActivity` +covers both, for the microphone and for pushed audio: + +```java +Subscription speaking = microphone.onVoiceActivity(active -> { + // On the event executor. }); -Subscription audio = microphone.onAudio(Duration.ofMillis(200), vad); ``` -The microphone's audio flows while a peer sends the track. For what the remote -peer sends, subscribe to its track: +## Audio track integration + +The examples module includes `VoiceActivityExample`, an adapter using +`StandaloneVoiceActivityDetector`. It buffers incoming audio into 10 ms frames, +converts mono or stereo input to 16 kHz mono with `AudioConverter`, and reports +each boolean decision. It also supports 44.1 kHz input. + +With the [Client API](/guide/client-api), it takes the `AudioChunk`s of a remote +track's `onAudio` subscription: ```java import dev.onvoid.webrtc.client.Subscription; @@ -180,10 +215,10 @@ finally { A microphone track does not: WebRTC passes captured audio from the audio device module straight to the senders, and the source of the track hands nothing to -sinks. Its audio after processing goes to the sink set with -`PeerConnectionFactory.setCapturedAudioSink`, which is what `LocalAudio.onAudio` -uses. An [`AudioRecorder`](/tools/audio/audio-recorder) captures a microphone -on its own, apart from any peer connection. +sinks. Its speech comes from a listener on the factory, see +[Speech events in native code](#speech-events-in-native-code). An +[`AudioRecorder`](/tools/audio/audio-recorder) captures a microphone on its +own, apart from any peer connection. The adapter invokes the consumer on the thread that delivers the audio: a thread of the `onAudio` subscription, or the audio callback thread of a sink. Keep work diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java index 5cc307f20..53b4c6bb9 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/VoiceActivityExample.java @@ -22,7 +22,6 @@ import java.util.Objects; import dev.onvoid.webrtc.client.AudioChunk; -import dev.onvoid.webrtc.client.LocalAudio; import dev.onvoid.webrtc.client.RemoteAudio; import dev.onvoid.webrtc.client.Subscription; import dev.onvoid.webrtc.media.audio.AudioConverter; @@ -30,11 +29,9 @@ import dev.onvoid.webrtc.media.audio.StandaloneVoiceActivityDetector; /** - * With the client API, subscribe it to the microphone with - * {@link LocalAudio#onAudio(Consumer)}, whose audio arrives after echo - * cancellation, or to a remote track with {@link RemoteAudio#onAudio(Consumer)}, - * then close the {@link Subscription} before close(). An AudioTrack of the - * native API takes it with addSink(), + * With the client API, subscribe it to a remote track with + * {@link RemoteAudio#onAudio(Consumer)}, then close the {@link Subscription} + * before close(). An AudioTrack of the native API takes it with addSink(), * then removeSink() before close(), if its source hands audio to sinks: that * of a remote track or a CustomAudioSource does, a microphone's does not. * Buffers arbitrary chunk lengths into 10 ms frames and converts interleaved diff --git a/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h b/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h index 9a0fefc34..8e8644ed3 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h +++ b/webrtc-jni/src/main/cpp/include/JNI_AudioProcessing.h @@ -7,14 +7,6 @@ #ifdef __cplusplus extern "C" { #endif - /* - * Class: dev_onvoid_webrtc_media_audio_AudioProcessing - * Method: setCapturedAudioSink - * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;)V - */ - JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_setCapturedAudioSink - (JNIEnv*, jobject, jobject); - /* * Class: dev_onvoid_webrtc_media_audio_AudioProcessing * Method: applyConfig diff --git a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h index ec9870fe5..616ed1461 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h +++ b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h @@ -31,6 +31,22 @@ extern "C" { JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addSinkInternal (JNIEnv *, jobject, jobject, jboolean); + /* + * Class: dev_onvoid_webrtc_media_audio_AudioTrack + * Method: addVoiceActivityListenerInternal + * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FI)J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addVoiceActivityListenerInternal + (JNIEnv *, jobject, jobject, jfloat, jint); + + /* + * Class: dev_onvoid_webrtc_media_audio_AudioTrack + * Method: removeVoiceActivityListenerInternal + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_removeVoiceActivityListenerInternal + (JNIEnv *, jobject, jlong); + /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack * Method: removeSinkInternal diff --git a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h index 93f485818..8995ce152 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h @@ -25,11 +25,19 @@ extern "C" { /* * Class: dev_onvoid_webrtc_PeerConnectionFactory - * Method: setCapturedAudioSink - * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;)V + * Method: addVoiceActivityListenerInternal + * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FI)J */ - JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_setCapturedAudioSink - (JNIEnv *, jobject, jobject); + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_addVoiceActivityListenerInternal + (JNIEnv *, jobject, jobject, jfloat, jint); + + /* + * Class: dev_onvoid_webrtc_PeerConnectionFactory + * Method: removeVoiceActivityListenerInternal + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_removeVoiceActivityListenerInternal + (JNIEnv *, jobject, jlong); /* * Class: dev_onvoid_webrtc_PeerConnectionFactory @@ -81,10 +89,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_PeerConnectionFactory - * Method: dispose + * Method: disposeInternal * Signature: ()V */ - JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_dispose + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_disposeInternal (JNIEnv *, jobject); /* diff --git a/webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h b/webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h index cd0f49da2..37125e382 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/AudioCaptureTap.h @@ -17,8 +17,10 @@ #ifndef JNI_WEBRTC_MEDIA_AUDIO_CAPTURE_TAP_H_ #define JNI_WEBRTC_MEDIA_AUDIO_CAPTURE_TAP_H_ +#include "media/audio/VoiceActivityListener.h" +#include "media/audio/VoiceDetector.h" + #include "api/audio/audio_processing.h" -#include "api/AudioTrackSink.h" #include #include @@ -27,13 +29,14 @@ namespace jni { - // Hands the capture signal of an AudioProcessing, after all of its - // processing, to a sink: what the senders send. The module calls it - // through its capture post-processor, which is put in when the module is - // built, so the tap does nothing until a sink is set. + // Detects speech in the capture signal of an AudioProcessing, after all + // of its processing: what the senders send. The module calls it through + // its capture post-processor, which is put in when the module is built, + // so the tap does nothing until a listener is added. All listeners share + // one detector; each decides with its own threshold and hangover. // // The tap is shared: by the post-processor, for as long as the module - // lives, and by the Java objects that set its sink, the module's and the + // lives, and by the Java objects that hold it, the module's and the // factory's, for as long as they live. None of them outlives it. class AudioCaptureTap { @@ -41,10 +44,13 @@ namespace jni // The capture post-processor to build an AudioProcessing with. static std::unique_ptr CreateProcessing(std::shared_ptr tap); - // Replaces the sink, or stops the delivery with nullptr. Once this - // returns, the sink replaced is no longer called. Must not be - // called from within the sink. - void SetSink(std::unique_ptr sink); + // Adds a listener, which hears of the speech from the next 10 ms + // on. Returns it, as the handle that removes it. + VoiceActivityListener * AddListener(std::unique_ptr listener); + + // Removes a listener added and deletes it. Once this returns, it is + // no longer called. Must not be called from within a listener. + void RemoveListener(VoiceActivityListener * listener); // Called on the capture thread. void Process(const webrtc::AudioBuffer & audio); @@ -53,7 +59,10 @@ namespace jni std::mutex mutex; // Guarded by mutex. - std::unique_ptr sink; + std::vector> listeners; + + // Guarded by mutex. + VoiceDetector detector; // Guarded by mutex. std::vector samples; diff --git a/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h new file mode 100644 index 000000000..b29a23c91 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h @@ -0,0 +1,69 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_AUDIO_VOICE_ACTIVITY_LISTENER_H_ +#define JNI_WEBRTC_MEDIA_AUDIO_VOICE_ACTIVITY_LISTENER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include +#include + +namespace jni +{ + // Decides from the voice probability of each 10 ms chunk when speech + // begins and ends, and tells a Java VoiceActivityListener of each change, + // and of nothing else. Speech begins with a chunk at or above the + // threshold, and ends once the chunks stayed below it for longer than the + // hangover. + class VoiceActivityListener + { + public: + VoiceActivityListener(JNIEnv * env, const JavaGlobalRef & listener, float threshold, + int hangoverMs); + ~VoiceActivityListener() = default; + + // Called on the thread that delivers the audio. + void Update(float probability); + + private: + class JavaVoiceActivityListenerClass : public JavaClass + { + public: + explicit JavaVoiceActivityListenerClass(JNIEnv * env); + + jmethodID onVoiceActivity; + }; + + private: + JavaGlobalRef listener; + + const std::shared_ptr javaClass; + + const float threshold; + + // The hangover in chunks, rounded up. + const int hangoverChunks; + + // The chunks below the threshold since speech was last detected. + int quietChunks; + + bool speaking; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivitySink.h b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivitySink.h new file mode 100644 index 000000000..9e7604f8f --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivitySink.h @@ -0,0 +1,49 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_AUDIO_VOICE_ACTIVITY_SINK_H_ +#define JNI_WEBRTC_MEDIA_AUDIO_VOICE_ACTIVITY_SINK_H_ + +#include "media/audio/VoiceActivityListener.h" +#include "media/audio/VoiceDetector.h" + +#include "api/media_stream_interface.h" + +#include + +namespace jni +{ + // A sink of an audio track that detects speech in what the source of the + // track hands to sinks, and tells a listener when it begins and ends. + class VoiceActivitySink : public webrtc::AudioTrackSinkInterface + { + public: + explicit VoiceActivitySink(std::unique_ptr listener); + ~VoiceActivitySink() = default; + + // AudioTrackSinkInterface implementation. The other variants + // forward to this one. + using webrtc::AudioTrackSinkInterface::OnData; + void OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames) override; + + private: + VoiceDetector detector; + + const std::unique_ptr listener; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/VoiceDetector.h b/webrtc-jni/src/main/cpp/include/media/audio/VoiceDetector.h new file mode 100644 index 000000000..866a5ced1 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/audio/VoiceDetector.h @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_AUDIO_VOICE_DETECTOR_H_ +#define JNI_WEBRTC_MEDIA_AUDIO_VOICE_DETECTOR_H_ + +#include "common_audio/resampler/include/push_resampler.h" +#include "modules/audio_processing/vad/voice_activity_detector.h" + +#include +#include +#include +#include +#include + +namespace jni +{ + // The voice probability of audio in 10 ms chunks, from WebRTC's + // VoiceActivityDetector. Chunks are mixed down to mono and resampled to + // the detector's 16 kHz here, at any rate, as the detector's own + // resampler stops the process on a rate it does not know, such as + // 44.1 kHz. Not thread-safe. + class VoiceDetector + { + public: + VoiceDetector(); + + // Takes 10 ms of 16-bit PCM with interleaved channels. Returns + // the voice probability, from 0 to 1, or nothing for a chunk that + // is not 10 ms long, and for the first 30 ms, which the detector + // collects before its first probability. + std::optional Process(const int16_t * data, int sampleRate, size_t channels, size_t frames); + + // Like Process, for mono audio. + std::optional ProcessMono(const int16_t * data, int sampleRate, size_t frames); + + // Forgets the audio so far, as for a new stream. + void Reset(); + + private: + std::unique_ptr detector; + + webrtc::PushResampler resampler; + + std::vector mono; + + std::array resampled; + + // The chunks processed since the last reset, up to the three the + // detector needs for a probability. + int chunks; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp index 7ad260dd4..4932f9321 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioProcessing.cpp @@ -25,7 +25,6 @@ #include "JavaUtils.h" #include "WebRTCContext.h" -#include "api/AudioTrackSink.h" #include "media/audio/AudioCaptureTap.h" #include "media/audio/AudioProcessing.h" #include "media/audio/AudioProcessingConfig.h" @@ -40,25 +39,6 @@ #include #include -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_setCapturedAudioSink -(JNIEnv * env, jobject caller, jobject jSink) -{ - try { - webrtc::AudioProcessing * apm = GetHandle(env, caller); - CHECK_HANDLE(apm); - - auto tap = GetHandle>(env, caller, "captureTapHandle"); - CHECK_HANDLE(tap); - - (*tap)->SetSink(jSink == nullptr - ? nullptr - : std::make_unique(env, jni::JavaGlobalRef(env, jSink), false)); - } - catch (...) { - ThrowCxxJavaException(env); - } -} - JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioProcessing_applyConfig (JNIEnv * env, jobject caller, jobject config) { diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp index c6bb14406..ac18b51b5 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp @@ -18,6 +18,7 @@ #include "api/AudioTrackSink.h" #include "JavaNullPointerException.h" #include "JavaUtils.h" +#include "media/audio/VoiceActivitySink.h" #include "api/media_stream_interface.h" @@ -60,6 +61,42 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_removeSinkI } } +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addVoiceActivityListenerInternal +(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jint hangoverMs) +{ + try { + webrtc::AudioTrackInterface * track = GetHandle(env, caller); + CHECK_HANDLEV(track, 0); + + auto sink = new jni::VoiceActivitySink(std::make_unique(env, + jni::JavaGlobalRef(env, jListener), threshold, hangoverMs)); + + track->AddSink(sink); + + return reinterpret_cast(sink); + } + catch (...) { + ThrowCxxJavaException(env); + return 0; + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_removeVoiceActivityListenerInternal +(JNIEnv * env, jobject caller, jlong sinkHandle) +{ + webrtc::AudioTrackInterface * track = GetHandle(env, caller); + CHECK_HANDLE(track); + + auto sink = reinterpret_cast(sinkHandle); + + if (sink != nullptr) { + // Once removed, the source no longer calls it. + track->RemoveSink(sink); + + delete sink; + } +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_setVolumeInternal (JNIEnv * env, jobject caller, jdouble volume) { diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index 5961a5480..122f3c824 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -33,8 +33,8 @@ #include "WebRTCContext.h" #include "api/audio/builtin_audio_processing_builder.h" -#include "api/AudioTrackSink.h" #include "media/audio/AudioCaptureTap.h" +#include "media/audio/VoiceActivityListener.h" #include "api/audio/create_audio_device_module.h" #include "api/create_modular_peer_connection_factory.h" #include "api/enable_media.h" @@ -205,7 +205,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize } } -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_dispose +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_disposeInternal (JNIEnv * env, jobject caller) { webrtc::PeerConnectionFactoryInterface * factory = GetHandle(env, caller); @@ -256,19 +256,32 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_dispose } } -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_setCapturedAudioSink -(JNIEnv * env, jobject caller, jobject jSink) +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_addVoiceActivityListenerInternal +(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jint hangoverMs) { try { - webrtc::PeerConnectionFactoryInterface * factory = GetHandle(env, caller); - CHECK_HANDLE(factory); + auto tap = GetHandle>(env, caller, "captureTapHandle"); + CHECK_HANDLEV(tap, 0); + + auto listener = std::make_unique(env, jni::JavaGlobalRef(env, jListener), + threshold, hangoverMs); + + return reinterpret_cast((*tap)->AddListener(std::move(listener))); + } + catch (...) { + ThrowCxxJavaException(env); + return 0; + } +} +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_removeVoiceActivityListenerInternal +(JNIEnv * env, jobject caller, jlong listenerHandle) +{ + try { auto tap = GetHandle>(env, caller, "captureTapHandle"); CHECK_HANDLE(tap); - (*tap)->SetSink(jSink == nullptr - ? nullptr - : std::make_unique(env, jni::JavaGlobalRef(env, jSink), false)); + (*tap)->RemoveListener(reinterpret_cast(listenerHandle)); } catch (...) { ThrowCxxJavaException(env); diff --git a/webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp b/webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp index 86bad01c2..ac4b695f1 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/AudioCaptureTap.cpp @@ -58,27 +58,46 @@ namespace jni return std::make_unique(std::move(tap)); } - void AudioCaptureTap::SetSink(std::unique_ptr next) + VoiceActivityListener * AudioCaptureTap::AddListener(std::unique_ptr listener) { - std::unique_ptr replaced; + std::lock_guard lock(mutex); + + if (listeners.empty()) { + // The audio it saw last may be long ago. + detector.Reset(); + } + + listeners.push_back(std::move(listener)); + + return listeners.back().get(); + } + + void AudioCaptureTap::RemoveListener(VoiceActivityListener * listener) + { + std::unique_ptr removed; { // Waits for a delivery in progress, which holds the lock. std::lock_guard lock(mutex); - replaced = std::move(sink); - sink = std::move(next); + for (auto it = listeners.begin(); it != listeners.end(); ++it) { + if (it->get() == listener) { + removed = std::move(*it); + listeners.erase(it); + break; + } + } } // Deleted outside the lock: it releases a Java reference. - replaced.reset(); + removed.reset(); } void AudioCaptureTap::Process(const webrtc::AudioBuffer & audio) { std::lock_guard lock(mutex); - if (!sink) { + if (listeners.empty()) { return; } @@ -86,17 +105,29 @@ namespace jni const size_t frames = audio.num_frames(); const float * const * data = audio.channels_const(); - samples.resize(channels * frames); + samples.resize(frames); // The buffer holds one channel after another, as floats in the range - // of 16-bit samples; a sink takes them interleaved. - for (size_t channel = 0; channel < channels; channel++) { - for (size_t frame = 0; frame < frames; frame++) { - samples[frame * channels + channel] = webrtc::FloatS16ToS16(data[channel][frame]); + // of 16-bit samples; the detector takes them mixed down to mono. + for (size_t frame = 0; frame < frames; frame++) { + float sum = 0; + + for (size_t channel = 0; channel < channels; channel++) { + sum += data[channel][frame]; } + + samples[frame] = webrtc::FloatS16ToS16(sum / static_cast(channels)); } // AudioProcessing works on 10 ms at a time. - sink->OnData(samples.data(), 16, static_cast(frames * 100), channels, frames); + std::optional probability = detector.ProcessMono(samples.data(), static_cast(frames * 100), frames); + + if (!probability) { + return; + } + + for (const auto & listener : listeners) { + listener->Update(*probability); + } } } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp new file mode 100644 index 000000000..2480084ad --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/VoiceActivityListener.h" +#include "api/WebRTCUtils.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +namespace jni +{ + VoiceActivityListener::VoiceActivityListener(JNIEnv * env, const JavaGlobalRef & listener, + float threshold, int hangoverMs) : + listener(listener), + javaClass(JavaClasses::get(env)), + threshold(threshold), + hangoverChunks((hangoverMs + 9) / 10), + quietChunks(0), + speaking(false) + { + } + + void VoiceActivityListener::Update(float probability) + { + if (probability >= threshold) { + quietChunks = 0; + + if (speaking) { + return; + } + + speaking = true; + } + else if (speaking && ++quietChunks > hangoverChunks) { + quietChunks = 0; + speaking = false; + } + else { + return; + } + + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return; + } + + env->CallVoidMethod(listener, javaClass->onVoiceActivity, static_cast(speaking)); + + ReportPendingException(env); + } + + VoiceActivityListener::JavaVoiceActivityListenerClass::JavaVoiceActivityListenerClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_AUDIO"VoiceActivityListener"); + + onVoiceActivity = GetMethod(env, cls, "onVoiceActivity", "(Z)V"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivitySink.cpp b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivitySink.cpp new file mode 100644 index 000000000..29468d05b --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivitySink.cpp @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/VoiceActivitySink.h" + +namespace jni +{ + VoiceActivitySink::VoiceActivitySink(std::unique_ptr listener) : + listener(std::move(listener)) + { + } + + void VoiceActivitySink::OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, + size_t frames) + { + if (bitsPerSample != 16) { + return; + } + + std::optional probability = detector.Process(static_cast(data), sampleRate, + channels, frames); + + if (probability) { + listener->Update(*probability); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/VoiceDetector.cpp b/webrtc-jni/src/main/cpp/src/media/audio/VoiceDetector.cpp new file mode 100644 index 000000000..0bb6f16dc --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/VoiceDetector.cpp @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/VoiceDetector.h" + +#include "api/audio/audio_view.h" + +namespace jni +{ + namespace + { + // The detector works on 10 ms at 16 kHz. + constexpr int kDetectorRate = 16000; + constexpr size_t kDetectorFrames = kDetectorRate / 100; + + // What PushResampler takes in 10 ms, which is 384 kHz. + constexpr size_t kMaxFrames = 3840; + + // The chunks the detector collects before its first probability. + constexpr int kWarmUpChunks = 3; + } + + VoiceDetector::VoiceDetector() : + detector(std::make_unique()), + chunks(0) + { + } + + std::optional VoiceDetector::Process(const int16_t * data, int sampleRate, size_t channels, size_t frames) + { + if (channels == 1) { + return ProcessMono(data, sampleRate, frames); + } + if (channels == 0 || frames > kMaxFrames) { + return std::nullopt; + } + + mono.resize(frames); + + for (size_t frame = 0; frame < frames; frame++) { + int sum = 0; + + for (size_t channel = 0; channel < channels; channel++) { + sum += data[frame * channels + channel]; + } + + mono[frame] = static_cast(sum / static_cast(channels)); + } + + return ProcessMono(mono.data(), sampleRate, frames); + } + + std::optional VoiceDetector::ProcessMono(const int16_t * data, int sampleRate, size_t frames) + { + if (frames == 0 || frames > kMaxFrames || static_cast(sampleRate) != frames * 100) { + return std::nullopt; + } + + const int16_t * chunk = data; + + if (frames != kDetectorFrames) { + resampler.Resample(webrtc::MonoView(data, frames), + webrtc::MonoView(resampled.data(), resampled.size())); + + chunk = resampled.data(); + } + + detector->ProcessChunk(chunk, kDetectorFrames, kDetectorRate); + + if (chunks < kWarmUpChunks && ++chunks < kWarmUpChunks) { + // Until then the detector reports its initial 1. + return std::nullopt; + } + + return detector->last_voice_probability(); + } + + void VoiceDetector::Reset() + { + detector = std::make_unique(); + chunks = 0; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index 8d7b47ad0..7869b7026 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -23,10 +23,11 @@ import dev.onvoid.webrtc.media.audio.AudioDeviceModuleBase; import dev.onvoid.webrtc.media.audio.AudioOptions; import dev.onvoid.webrtc.media.audio.AudioProcessing; -import dev.onvoid.webrtc.media.audio.AudioTrackSink; import dev.onvoid.webrtc.media.audio.AudioTrackSource; import dev.onvoid.webrtc.media.audio.AudioTrack; import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.audio.VoiceActivityConfig; +import dev.onvoid.webrtc.media.audio.VoiceActivityListener; import dev.onvoid.webrtc.media.audio.codec.AudioDecoderFactory; import dev.onvoid.webrtc.media.audio.codec.AudioEncoderFactory; import dev.onvoid.webrtc.media.audio.codec.BuiltinAudioDecoderFactory; @@ -40,6 +41,7 @@ import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; +import java.util.IdentityHashMap; import java.util.Map; import java.util.Objects; @@ -106,6 +108,12 @@ private enum AudioInputMode { @SuppressWarnings("unused") private long captureTapHandle; + /** + * The listeners added to the audio processing, which may outlive the + * factory; guarded by itself. + */ + private final Map voiceActivityListeners = new IdentityHashMap<>(); + /** Guards {@link #audioInputMode}. */ private final Object audioInputLock = new Object(); @@ -412,27 +420,79 @@ public RTCPeerConnection createPeerConnection(RTCConfiguration config, public native RTCRtpCapabilities getRtpSenderCapabilities(MediaType type); /** - * Sets the sink that gets the microphone audio after audio processing, - * with echo cancellation, noise suppression and gain control applied as - * configured: the audio the senders send. It comes from the {@link - * AudioProcessing} the factory was built with, or else from the one the - * factory built itself, and is the same sink as that module's {@link - * AudioProcessing#setCapturedAudioSink}. + * Adds a listener that hears when speech begins and ends in the + * microphone audio after audio processing, with echo cancellation, noise + * suppression and gain control applied as configured: the audio the + * senders send, so that the remote peer's voice from the speakers does not + * count as this side speaking. The audio is analyzed in native code, in + * the {@link AudioProcessing} the factory was built with, or else in the + * one the factory built itself, and only the changes reach the listener. *

* Audio arrives while the audio device module records, which WebRTC starts - * once a peer connection sends a track of {@link #createAudioSource}. Each - * call holds 10 ms of 16-bit PCM with interleaved channels, at the rate - * the module processes at. The sink runs on WebRTC's capture thread, so it - * must return promptly, and it must not set the sink itself. Audio pushed - * through a {@link CustomAudioSource} does not pass audio processing and - * does not arrive. - * - * @param sink The sink, or {@code null} to stop the delivery. + * once a peer connection sends a track of {@link #createAudioSource}. The + * listener runs on WebRTC's capture thread. Audio pushed through a {@link + * CustomAudioSource} does not pass audio processing: its speech comes from + * {@link AudioTrack#addVoiceActivityListener}. + * + * @param config when audio counts as speech. + * @param listener the listener. Adding it again does nothing. + */ + public void addVoiceActivityListener(VoiceActivityConfig config, VoiceActivityListener listener) { + Objects.requireNonNull(config, "config"); + Objects.requireNonNull(listener, "listener"); + + synchronized (voiceActivityListeners) { + if (voiceActivityListeners.containsKey(listener)) { + return; + } + + long nativeListener = addVoiceActivityListenerInternal(listener, (float) config.getThreshold(), + (int) config.getHangover().toMillis()); + + voiceActivityListeners.put(listener, nativeListener); + } + } + + /** + * Removes a listener added with {@link #addVoiceActivityListener}. Once + * this returns, it is no longer called. If it was not added, this is a + * no-op. + * + * @param listener the listener. */ - public native void setCapturedAudioSink(AudioTrackSink sink); + public void removeVoiceActivityListener(VoiceActivityListener listener) { + Objects.requireNonNull(listener, "listener"); + + synchronized (voiceActivityListeners) { + Long nativeListener = voiceActivityListeners.remove(listener); + + if (nativeListener != null) { + removeVoiceActivityListenerInternal(nativeListener); + } + } + } @Override - public native void dispose(); + public void dispose() { + // An AudioProcessing of the application lives on with its listeners. + synchronized (voiceActivityListeners) { + for (long nativeListener : voiceActivityListeners.values()) { + removeVoiceActivityListenerInternal(nativeListener); + } + + voiceActivityListeners.clear(); + } + + disposeInternal(); + } + + private native void disposeInternal(); + + private native long addVoiceActivityListenerInternal(VoiceActivityListener listener, float threshold, + int hangoverMs); + + private native void removeVoiceActivityListenerInternal(long listenerHandle); + /** * Creates the native factory. A null codec factory stands for the diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java deleted file mode 100644 index cbbed3bf6..000000000 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/CapturedAudio.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2026 Alex Andres - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dev.onvoid.webrtc.client; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -import dev.onvoid.webrtc.PeerConnectionFactory; -import dev.onvoid.webrtc.media.audio.AudioTrackSink; - -/** - * Hands the microphone audio of a runtime, after audio processing, to the - * audio subscriptions of its microphone tracks. The factory's audio - * processing calls it on WebRTC's capture thread, and only while one is - * subscribed, so that the capture thread does not call into Java for nothing. - */ -final class CapturedAudio implements AudioTrackSink { - - private final PeerConnectionFactory factory; - - /** Changed under this object's lock, read on the capture thread. */ - private final List sinks = new CopyOnWriteArrayList<>(); - - - CapturedAudio(PeerConnectionFactory factory) { - this.factory = factory; - } - - synchronized void add(AudioTrackSink sink) { - if (sinks.isEmpty()) { - factory.setCapturedAudioSink(this); - } - - sinks.add(sink); - } - - synchronized void remove(AudioTrackSink sink) { - if (sinks.remove(sink) && sinks.isEmpty()) { - factory.setCapturedAudioSink(null); - } - } - - @Override - public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { - // Each sink copies what it keeps. - for (AudioTrackSink sink : sinks) { - sink.onData(data, bitsPerSample, sampleRate, channels, frames); - } - } -} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java index cb37a98b2..fbdda3520 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java @@ -16,45 +16,44 @@ package dev.onvoid.webrtc.client; -import java.time.Duration; import java.util.function.Consumer; +import dev.onvoid.webrtc.media.audio.VoiceActivityConfig; + /** * An audio track this side sends. */ public interface LocalAudio extends LocalTrack { /** - * Calls the handler with the audio this track sends, in order, such as - * for a voice activity detector or a level meter. Pushed audio arrives as - * pushed. A microphone's audio arrives after audio processing, with echo - * cancellation, noise suppression and gain control applied as configured, - * so that the remote peer's voice from the speakers does not count as - * this side speaking. It flows while WebRTC records, which it does while - * a peer sends a microphone track, and every microphone track of a - * runtime has the same, as WebRTC records one microphone for all of them. - *

- * A handler that falls behind gets the audio of the last {@link - * WebRtc.Builder#audioBuffer(Duration) audio buffer} of the runtime; the - * oldest audio beyond that is dropped. + * Like {@link #onVoiceActivity(VoiceActivityConfig, Consumer)}, with the + * default threshold and hangover of {@link VoiceActivityConfig}. * - * @param handler called on a thread of this subscription. + * @param handler called on the runtime's event executor, in order. * - * @return the subscription that stops the audio. + * @return the subscription that stops the calls. */ - Subscription onAudio(Consumer handler); + Subscription onVoiceActivity(Consumer handler); /** - * Like {@link #onAudio(Consumer)}, with a buffer of its own: a handler - * that falls behind gets the audio of the last {@code buffer}, and the - * oldest audio beyond that is dropped. A buffer shorter than one chunk - * keeps only the newest chunk. + * Calls the handler when this side begins to speak, with {@code true}, and + * when it stops, with {@code false}. The audio is analyzed in native code, + * and only these changes reach Java. Pushed audio is analyzed as pushed. A + * microphone's audio is analyzed after audio processing, with echo + * cancellation, noise suppression and gain control applied as configured, + * so that the remote peer's voice from the speakers does not count as + * this side speaking. It flows while WebRTC records, which it does while a + * peer sends a microphone track, and every microphone track of a runtime + * hears the same, as WebRTC records one microphone for all of them. + *

+ * Closing the subscription or the track while speech goes on calls the + * handler no more, so it does not hear the end of that speech. * - * @param buffer the most audio that waits for the handler. - * @param handler called on a thread of this subscription. + * @param config when audio counts as speech. + * @param handler called on the runtime's event executor, in order. * - * @return the subscription that stops the audio. + * @return the subscription that stops the calls. */ - Subscription onAudio(Duration buffer, Consumer handler); + Subscription onVoiceActivity(VoiceActivityConfig config, Consumer handler); } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java index 720d431fd..a1a872afa 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java @@ -18,20 +18,20 @@ import static java.util.Objects.requireNonNull; -import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; -import dev.onvoid.webrtc.internal.Timeouts; import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.VoiceActivityConfig; +import dev.onvoid.webrtc.media.audio.VoiceActivityListener; /** - * A local audio track, from the microphone or pushed audio. Each audio - * subscription has a thread of its own. Pushed audio reaches it as a sink of - * the native track. A microphone's source hands no audio to sinks, as WebRTC + * A local audio track, from the microphone or pushed audio. Speech is + * detected in native code. Pushed audio reaches the detector as a sink of the + * native track. A microphone's source hands no audio to sinks, as WebRTC * passes captured audio straight to the senders, so a microphone track takes - * it from the runtime's audio processing instead. + * its speech from the runtime's audio processing instead. */ class LocalAudioImpl extends LocalTrackImpl implements LocalAudio { @@ -39,7 +39,7 @@ class LocalAudioImpl extends LocalTrackImpl implements LocalAudio { // Guarded by this. - private final List dispatchers = new ArrayList<>(); + private final List listeners = new ArrayList<>(); /** @@ -53,34 +53,39 @@ class LocalAudioImpl extends LocalTrackImpl implements LocalAudio { } @Override - public Subscription onAudio(Consumer handler) { - return onAudio(rtc.audioBuffer, handler); + public Subscription onVoiceActivity(Consumer handler) { + return onVoiceActivity(new VoiceActivityConfig(), handler); } @Override - public Subscription onAudio(Duration buffer, Consumer handler) { - Timeouts.requirePositive(buffer, "buffer"); + public Subscription onVoiceActivity(VoiceActivityConfig config, Consumer handler) { + requireNonNull(config, "config"); requireNonNull(handler, "handler"); - SerialExecutor executor = new SerialExecutor(rtc.framePool, rtc::reportError, false, null); - AudioDispatcher dispatcher = new AudioDispatcher(handler, buffer, executor, rtc.audioWork, - rtc::reportError); + VoiceActivity listener = new VoiceActivity(handler); synchronized (this) { if (!isLive()) { return () -> { }; } - attach(dispatcher); - dispatchers.add(dispatcher); + if (microphone) { + rtc.factory.addVoiceActivityListener(config, listener); + } + else { + audioTrack().addVoiceActivityListener(config, listener); + } + + listeners.add(listener); } return () -> { - dispatcher.close(); + listener.closed = true; synchronized (LocalAudioImpl.this) { - if (dispatchers.remove(dispatcher) && isLive()) { - detach(dispatcher); + // Once the track ended, the release removes what is left. + if (isLive() && listeners.remove(listener)) { + detach(listener); } } }; @@ -88,38 +93,60 @@ public Subscription onAudio(Duration buffer, Consumer handler) { @Override void beforeRelease() { - List attached; + List attached; synchronized (this) { - attached = new ArrayList<>(dispatchers); - dispatchers.clear(); + attached = new ArrayList<>(listeners); + listeners.clear(); } - for (AudioDispatcher dispatcher : attached) { - detach(dispatcher); - dispatcher.close(); + for (VoiceActivity listener : attached) { + listener.closed = true; + detach(listener); } } - private void attach(AudioDispatcher dispatcher) { + private void detach(VoiceActivity listener) { if (microphone) { - rtc.capturedAudio.add(dispatcher); + rtc.factory.removeVoiceActivityListener(listener); } else { - audioTrack().addSink(dispatcher); - } - } - - private void detach(AudioDispatcher dispatcher) { - if (microphone) { - rtc.capturedAudio.remove(dispatcher); - } - else { - audioTrack().removeSink(dispatcher); + audioTrack().removeVoiceActivityListener(listener); } } private AudioTrack audioTrack() { return (AudioTrack) nativeTrack(); } + + + + /** + * Hands the changes, which arrive on the thread that delivers the audio, + * to the event executor, so that no handler holds up that thread. + */ + private final class VoiceActivity implements VoiceActivityListener { + + private final Consumer handler; + + volatile boolean closed; + + + VoiceActivity(Consumer handler) { + this.handler = handler; + } + + @Override + public void onVoiceActivity(boolean speaking) { + if (closed) { + return; + } + + media.events.execute(() -> { + if (!closed) { + handler.accept(speaking); + } + }); + } + } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java index e64b7d08f..ec06212bc 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java @@ -135,8 +135,6 @@ public final class WebRtc implements AutoCloseable { /** Created when the runtime was built with an audio processing config. */ final AudioProcessing audioProcessing; - final CapturedAudio capturedAudio; - private final Duration closeTimeout; private final Consumer errorHandler; @@ -263,7 +261,6 @@ private WebRtc(Builder builder) { this.audioModule = module; this.audioProcessing = processing; this.factory = created; - this.capturedAudio = new CapturedAudio(created); this.media = new LocalMediaImpl(this, module); } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java index 1df8d67ef..c7fecb929 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioProcessing.java @@ -159,22 +159,6 @@ public native int processReverseStream(byte[] src, AudioProcessingStreamConfig inputConfig, AudioProcessingStreamConfig outputConfig, byte[] dest); - /** - * Sets the sink that gets the capture stream after all processing, as - * {@link #processStream} puts it out, or, for a module a {@link - * dev.onvoid.webrtc.PeerConnectionFactory} was built with, the microphone - * audio its senders send. Each call holds 10 ms of 16-bit PCM with - * interleaved channels, at the rate and channel count the module processes - * at, which may differ from the format {@code processStream} puts out. - *

- * The sink runs on the thread that processes the capture stream, WebRTC's - * capture thread for a factory, so it must return promptly, and it must - * not set the sink itself. - * - * @param sink The sink, or {@code null} to stop the delivery. - */ - public native void setCapturedAudioSink(AudioTrackSink sink); - @Override public native void dispose(); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java index 68a299450..7859915ae 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java @@ -29,6 +29,8 @@ public class AudioTrack extends MediaStreamTrack { private final Map sinks = new IdentityHashMap<>(); + private final Map voiceActivityListeners = new IdentityHashMap<>(); + private AudioTrack() { super(); @@ -42,6 +44,12 @@ public void dispose() { sinks.clear(); + for (long nativeSink : voiceActivityListeners.values()) { + removeVoiceActivityListenerInternal(nativeSink); + } + + voiceActivityListeners.clear(); + super.dispose(); } @@ -93,6 +101,49 @@ public void removeSink(AudioTrackSink sink) { } } + /** + * Adds a listener that hears when speech begins and ends in the audio of + * the track. The audio is analyzed in native code, and only the changes + * reach the listener, on the thread that delivers the audio. Like a sink, + * it gets what the source of the track hands to sinks: that of a remote + * track or a {@link CustomAudioSource} does, a microphone's does not. The + * microphone's speech comes from {@link + * dev.onvoid.webrtc.PeerConnectionFactory#addVoiceActivityListener}. + * + * @param config when audio counts as speech. + * @param listener the listener. Adding it again does nothing. + */ + public void addVoiceActivityListener(VoiceActivityConfig config, VoiceActivityListener listener) { + if (isNull(config) || isNull(listener)) { + throw new NullPointerException(); + } + if (voiceActivityListeners.containsKey(listener)) { + return; + } + + final long nativeSink = addVoiceActivityListenerInternal(listener, (float) config.getThreshold(), + (int) config.getHangover().toMillis()); + + voiceActivityListeners.put(listener, nativeSink); + } + + /** + * Removes a listener added with {@link #addVoiceActivityListener}. Once + * this returns, it is no longer called. If it was not added, this is a + * no-op. + */ + public void removeVoiceActivityListener(VoiceActivityListener listener) { + if (isNull(listener)) { + throw new NullPointerException(); + } + + final Long nativeSink = voiceActivityListeners.remove(listener); + + if (nonNull(nativeSink)) { + removeVoiceActivityListenerInternal(nativeSink); + } + } + /** * Get the signal level from the audio track. * @@ -125,4 +176,9 @@ public void setVolume(double volume) { private native void removeSinkInternal(long sinkHandle); + private native long addVoiceActivityListenerInternal(VoiceActivityListener listener, float threshold, + int hangoverMs); + + private native void removeVoiceActivityListenerInternal(long sinkHandle); + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityConfig.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityConfig.java new file mode 100644 index 000000000..ab9965f09 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityConfig.java @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import static java.util.Objects.requireNonNull; + +import java.time.Duration; + +/** + * When a {@link VoiceActivityListener} hears of speech. WebRTC's {@link + * VoiceActivityDetector} rates the audio with the probability that it holds + * a voice, from 0 to 1, anew every 30 ms; silence rates 0.01. Speech begins + * with 10 ms rated at or above the threshold, and ends once the audio stayed + * below it for longer than the hangover, which bridges the pauses between + * words. + *

+ * A lower threshold catches quiet speech sooner and takes more noise for + * speech; a longer hangover ends speech later and splits it less often. + */ +public final class VoiceActivityConfig { + + /** The threshold by default. */ + public static final double DEFAULT_THRESHOLD = 0.5; + + /** The hangover by default. */ + public static final Duration DEFAULT_HANGOVER = Duration.ofMillis(300); + + private final double threshold; + + private final Duration hangover; + + + /** + * Creates a config with the {@link #DEFAULT_THRESHOLD} and the {@link + * #DEFAULT_HANGOVER}. + */ + public VoiceActivityConfig() { + this(DEFAULT_THRESHOLD, DEFAULT_HANGOVER); + } + + /** + * @param threshold the voice probability, above 0 and at most 1, from + * which on audio counts as speech. + * @param hangover how long speech goes on after the audio fell below the + * threshold, zero or more; rounded up to whole 10 ms. + * + * @throws IllegalArgumentException if either is out of range. + */ + public VoiceActivityConfig(double threshold, Duration hangover) { + requireNonNull(hangover, "hangover"); + + if (!(threshold > 0 && threshold <= 1)) { + throw new IllegalArgumentException("The threshold must be above 0 and at most 1, got " + threshold); + } + if (hangover.isNegative() || hangover.compareTo(Duration.ofMillis(Integer.MAX_VALUE)) > 0) { + throw new IllegalArgumentException("The hangover must be zero or more and fit in an int of " + + "milliseconds, got " + hangover); + } + + this.threshold = threshold; + this.hangover = hangover; + } + + /** + * @return the voice probability from which on audio counts as speech. + */ + public double getThreshold() { + return threshold; + } + + /** + * @return how long speech goes on after the audio fell below the + * threshold. + */ + public Duration getHangover() { + return hangover; + } + + @Override + public String toString() { + return "VoiceActivityConfig[threshold=" + threshold + ", hangover=" + hangover + "]"; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityListener.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityListener.java new file mode 100644 index 000000000..54f134720 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/VoiceActivityListener.java @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +/** + * Hears when speech begins and ends in an audio stream, as a native voice + * activity detector finds it with a {@link VoiceActivityConfig}. The audio + * stays in native code; only the changes reach Java. + * + * @see dev.onvoid.webrtc.PeerConnectionFactory#addVoiceActivityListener + * @see AudioTrack#addVoiceActivityListener + */ +@FunctionalInterface +public interface VoiceActivityListener { + + /** + * Called when speech begins, with {@code true}, and when it ends, with + * {@code false}, alternating and starting with {@code true}. It runs on + * the thread that delivers the audio, such as WebRTC's capture thread, so + * it must return promptly, and it must not add or remove a listener. + * + * @param speaking whether speech began or ended. + */ + void onVoiceActivity(boolean speaking); + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java index e342ed178..266861fbe 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/LocalAudioTests.java @@ -16,97 +16,90 @@ package dev.onvoid.webrtc.client; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static dev.onvoid.webrtc.media.audio.SpeechFixture.concat; +import static dev.onvoid.webrtc.media.audio.SpeechFixture.silence; +import static dev.onvoid.webrtc.media.audio.SpeechFixture.speech; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import java.time.Duration; import java.util.Arrays; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import dev.onvoid.webrtc.media.audio.AudioOptions; +import dev.onvoid.webrtc.media.audio.AudioProcessing; import dev.onvoid.webrtc.media.audio.AudioProcessingConfig; import dev.onvoid.webrtc.media.audio.AudioProcessingStreamConfig; import dev.onvoid.webrtc.media.audio.AudioTrackSource; +import dev.onvoid.webrtc.media.audio.VoiceActivityConfig; import org.junit.jupiter.api.Test; /** - * The audio a local track sends: pushed audio as pushed, a microphone's after - * the runtime's audio processing. + * Speech in the audio a local track sends, detected in native code: pushed + * audio as pushed, a microphone's after the runtime's audio processing. */ class LocalAudioTests extends ClientTestBase { - private static final PcmFormat FORMAT = PcmFormat.of(48000, 1); + /** Long enough to bridge every pause of the fixture. */ + private static final VoiceActivityConfig ONE_SEGMENT = new VoiceActivityConfig(0.5, Duration.ofMillis(500)); @Test - void pushedAudioArrives() throws Exception { + void pushedSpeech() throws Exception { CustomAudio audio = rtc.media().customAudio("audio"); - CompletableFuture received = new CompletableFuture<>(); + BlockingQueue changes = new LinkedBlockingQueue<>(); - audio.onAudio(received::complete); - audio.push(constant(4800, 1000), FORMAT); + audio.onVoiceActivity(ONE_SEGMENT, changes::add); + // Sent, and with it analyzed, at the pace of real time. + audio.push(concat(speech(48000, 1), silence(1000, 48000, 1)), PcmFormat.of(48000, 1)); - AudioChunk chunk = await(received); - - assertEquals(48000, chunk.sampleRate()); - assertEquals(1, chunk.channels()); - assertSamples(1000, chunk); + assertEquals(Boolean.TRUE, changes.poll(10, TimeUnit.SECONDS)); + assertEquals(Boolean.FALSE, changes.poll(10, TimeUnit.SECONDS)); } @Test - void microphoneAudioAfterProcessing() throws Exception { - // A fixed gain makes what was processed differ from what went in. - AudioProcessingConfig config = new AudioProcessingConfig(); - config.gainControllerDigital.enabled = true; - config.gainControllerDigital.fixedDigital.gainDb = 6; - + void microphoneSpeechAfterProcessing() throws Exception { WebRtc runtime = WebRtc.builder() .audio(AudioMode.HEADLESS) - .audioProcessing(config) + .audioProcessing(new AudioProcessingConfig()) .errorHandler(errors::add) .build(); try { LocalAudio microphone = microphone(runtime); - CompletableFuture first = new CompletableFuture<>(); - CompletableFuture second = new CompletableFuture<>(); + BlockingQueue first = new LinkedBlockingQueue<>(); + BlockingQueue second = new LinkedBlockingQueue<>(); - Subscription one = microphone.onAudio(first::complete); - microphone.onAudio(second::complete); + Subscription one = microphone.onVoiceActivity(ONE_SEGMENT, first::add); + microphone.onVoiceActivity(ONE_SEGMENT, second::add); // Stands in for WebRTC, which hands the microphone's audio to the // same audio processing before it sends it. - byte[] input = constant(480, 1000); - byte[] output = new byte[input.length]; - AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); - - assertEquals(0, runtime.audioProcessing.processStream(input, format, format, output)); + process(runtime.audioProcessing, concat(speech(48000, 1), silence(1000, 48000, 1))); - AudioChunk chunk = await(first); + assertEquals(Boolean.TRUE, first.poll(10, TimeUnit.SECONDS)); + assertEquals(Boolean.FALSE, first.poll(10, TimeUnit.SECONDS)); + assertEquals(Boolean.TRUE, second.poll(10, TimeUnit.SECONDS)); + assertEquals(Boolean.FALSE, second.poll(10, TimeUnit.SECONDS)); - assertEquals(48000, chunk.sampleRate()); - assertEquals(480, chunk.frames()); - assertArrayEquals(output, chunk.samples()); - assertFalse(Arrays.equals(input, chunk.samples())); - assertArrayEquals(output, await(second).samples()); - - // The other subscription keeps the audio coming. + // The other subscription keeps hearing. one.close(); + process(runtime.audioProcessing, concat(speech(48000, 1), silence(1000, 48000, 1))); - CompletableFuture third = new CompletableFuture<>(); - microphone.onAudio(third::complete); - runtime.audioProcessing.processStream(input, format, format, output); - - assertArrayEquals(output, await(third).samples()); + assertEquals(Boolean.TRUE, second.poll(10, TimeUnit.SECONDS)); + assertEquals(Boolean.FALSE, second.poll(10, TimeUnit.SECONDS)); + // The same executor would have called it before. + assertNull(first.poll()); // Closing the track ends its subscriptions. await(microphone.closeAsync()); - Subscription late = microphone.onAudio(chunkAfter -> { }); + Subscription late = microphone.onVoiceActivity(speaking -> { }); late.close(); } finally { @@ -120,15 +113,40 @@ void microphoneWithFactorysOwnProcessing() throws Exception { // subscribing and closing needs nothing of the application. LocalAudio microphone = microphone(rtc); - Subscription audio = microphone.onAudio(chunk -> { }); - audio.close(); - microphone.onAudio(chunk -> { }); + Subscription activity = microphone.onVoiceActivity(speaking -> { }); + activity.close(); + microphone.onVoiceActivity(ONE_SEGMENT, speaking -> { }); await(microphone.closeAsync()); assertTrue(microphone.isEnded()); } + @Test + void closingTrackEndsSubscriptions() throws Exception { + CustomAudio audio = rtc.media().customAudio("audio"); + BlockingQueue changes = new LinkedBlockingQueue<>(); + + Subscription activity = audio.onVoiceActivity(ONE_SEGMENT, changes::add); + + await(audio.closeAsync()); + + // Closing it after the track does nothing more. + activity.close(); + audio.onVoiceActivity(changes::add).close(); + + assertTrue(changes.isEmpty()); + } + + @Test + void rejectsNull() { + CustomAudio audio = rtc.media().customAudio("audio"); + + assertThrows(NullPointerException.class, () -> audio.onVoiceActivity(null)); + assertThrows(NullPointerException.class, () -> audio.onVoiceActivity(null, speaking -> { })); + assertThrows(NullPointerException.class, () -> audio.onVoiceActivity(ONE_SEGMENT, null)); + } + /** * A microphone track of a headless runtime, which records nothing, so * that the test hands its audio processing the audio. @@ -141,22 +159,14 @@ private static LocalAudio microphone(WebRtc runtime) { runtime.factory.createAudioTrack("microphone", source), source::dispose, true)); } - /** Mono 16-bit PCM with every sample set to the value. */ - private static byte[] constant(int frames, int value) { - ByteBuffer pcm = ByteBuffer.allocate(frames * 2).order(ByteOrder.LITTLE_ENDIAN); - - for (int i = 0; i < frames; i++) { - pcm.putShort((short) value); - } - - return pcm.array(); - } - - private static void assertSamples(int value, AudioChunk chunk) { - ByteBuffer samples = ByteBuffer.wrap(chunk.samples()).order(ByteOrder.LITTLE_ENDIAN); + /** Processes 48 kHz mono in 10 ms chunks. */ + private static void process(AudioProcessing processing, byte[] pcm) { + AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); + byte[] output = new byte[960]; - while (samples.hasRemaining()) { - assertEquals(value, samples.getShort()); + for (int offset = 0; offset + 960 <= pcm.length; offset += 960) { + assertEquals(0, processing.processStream(Arrays.copyOfRange(pcm, offset, offset + 960), format, + format, output)); } } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java deleted file mode 100644 index b263cfef6..000000000 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CapturedAudioTest.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2026 Alex Andres - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dev.onvoid.webrtc.media.audio; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -import dev.onvoid.webrtc.PeerConnectionFactory; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * The capture stream of an {@link AudioProcessing} after all processing, - * taken by a sink, which is what a factory's senders send. - */ -class CapturedAudioTest { - - private final List captured = new CopyOnWriteArrayList<>(); - - private final AudioTrackSink sink = (data, bitsPerSample, sampleRate, channels, frames) -> - captured.add(new Captured(data, bitsPerSample, sampleRate, channels, frames)); - - private AudioProcessing processing; - - - @BeforeEach - void createProcessing() { - processing = new AudioProcessing(); - } - - @AfterEach - void disposeProcessing() { - processing.dispose(); - } - - @Test - void sinkGetsProcessedStream() { - // A fixed gain makes what was processed differ from what went in. - AudioProcessingConfig config = new AudioProcessingConfig(); - config.gainControllerDigital.enabled = true; - config.gainControllerDigital.fixedDigital.gainDb = 6; - processing.applyConfig(config); - processing.setCapturedAudioSink(sink); - - AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); - byte[] input = tone(48000, 1); - byte[] output = new byte[input.length]; - - assertEquals(0, processing.processStream(input, format, format, output)); - - assertEquals(1, captured.size()); - - Captured chunk = captured.get(0); - - assertEquals(16, chunk.bitsPerSample); - assertEquals(48000, chunk.sampleRate); - assertEquals(1, chunk.channels); - assertEquals(480, chunk.frames); - // Exactly what the stream puts out, after the gain. - assertArrayEquals(output, chunk.data); - assertFalse(Arrays.equals(input, chunk.data)); - } - - @Test - void sinkGetsProcessingRate() { - processing.setCapturedAudioSink(sink); - - AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(16000, 1); - byte[] input = tone(16000, 1); - - processing.processStream(input, format, format, new byte[input.length]); - - Captured chunk = captured.get(0); - - assertEquals(16000, chunk.sampleRate); - assertEquals(160, chunk.frames); - assertEquals(chunk.frames * chunk.channels * 2, chunk.data.length); - } - - @Test - void sinkRemoved() { - AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); - byte[] input = tone(48000, 1); - - // None set yet. - processing.processStream(input, format, format, new byte[input.length]); - - processing.setCapturedAudioSink(sink); - processing.processStream(input, format, format, new byte[input.length]); - - processing.setCapturedAudioSink(null); - processing.processStream(input, format, format, new byte[input.length]); - - assertEquals(1, captured.size()); - } - - @Test - void factoryTakesSinkOfItsProcessing() { - HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); - PeerConnectionFactory factory = new PeerConnectionFactory(module, processing); - - try { - factory.setCapturedAudioSink(sink); - - // What the factory's audio processing processes, as it would the - // microphone's audio. - AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(48000, 1); - byte[] input = tone(48000, 1); - processing.processStream(input, format, format, new byte[input.length]); - - assertEquals(1, captured.size()); - - factory.setCapturedAudioSink(null); - processing.processStream(input, format, format, new byte[input.length]); - - assertEquals(1, captured.size()); - } - finally { - factory.dispose(); - module.dispose(); - } - } - - @Test - void factoryWithOwnProcessing() { - HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); - PeerConnectionFactory factory = new PeerConnectionFactory(module); - - try { - factory.setCapturedAudioSink(sink); - factory.setCapturedAudioSink(null); - factory.setCapturedAudioSink(sink); - } - finally { - // Releases the sink still set along with the module. - factory.dispose(); - module.dispose(); - } - - assertThrows(NullPointerException.class, () -> factory.setCapturedAudioSink(sink)); - } - - @Test - void processingOutlivesItsObject() { - // The factory keeps the module, and with it the sink, after the - // application disposed of its object. - AudioProcessing shared = new AudioProcessing(); - HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); - PeerConnectionFactory factory = new PeerConnectionFactory(module, shared); - - try { - shared.dispose(); - - factory.setCapturedAudioSink(sink); - factory.setCapturedAudioSink(null); - - assertThrows(NullPointerException.class, () -> shared.setCapturedAudioSink(sink)); - } - finally { - factory.dispose(); - module.dispose(); - } - } - - /** 10 ms of a 440 Hz tone. */ - private static byte[] tone(int sampleRate, int channels) { - int frames = sampleRate / 100; - byte[] pcm = new byte[frames * channels * 2]; - - for (int i = 0; i < frames; i++) { - short sample = (short) (Math.sin(2 * Math.PI * 440 * i / sampleRate) * 4000); - - for (int channel = 0; channel < channels; channel++) { - int offset = (i * channels + channel) * 2; - pcm[offset] = (byte) sample; - pcm[offset + 1] = (byte) (sample >> 8); - } - } - - return pcm; - } - - - - private static final class Captured { - - final byte[] data; - final int bitsPerSample; - final int sampleRate; - final int channels; - final int frames; - - - Captured(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { - this.data = data.clone(); - this.bitsPerSample = bitsPerSample; - this.sampleRate = sampleRate; - this.channels = channels; - this.frames = frames; - } - } -} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SpeechFixture.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SpeechFixture.java new file mode 100644 index 000000000..699b97b93 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SpeechFixture.java @@ -0,0 +1,100 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * The speech fixture of {@code /audio/vad-speech-16k.pcm}, at any rate and + * channel count, and silence to follow it. + */ +public final class SpeechFixture { + + private static final short[] SPEECH = load(); + + + private SpeechFixture() { + } + + /** + * One second of synthetic speech at another rate, by linear + * interpolation, with every channel the same. + * + * @return 16-bit little-endian PCM with interleaved channels. + */ + public static byte[] speech(int sampleRate, int channels) { + int frames = (int) ((long) SPEECH.length * sampleRate / 16000); + ByteBuffer pcm = ByteBuffer.allocate(frames * channels * 2).order(ByteOrder.LITTLE_ENDIAN); + + for (int frame = 0; frame < frames; frame++) { + double position = (double) frame * 16000 / sampleRate; + int index = (int) position; + double fraction = position - index; + int next = Math.min(index + 1, SPEECH.length - 1); + short sample = (short) Math.round(SPEECH[index] * (1 - fraction) + SPEECH[next] * fraction); + + for (int channel = 0; channel < channels; channel++) { + pcm.putShort(sample); + } + } + + return pcm.array(); + } + + /** + * @return silence as 16-bit PCM with interleaved channels. + */ + public static byte[] silence(int millis, int sampleRate, int channels) { + return new byte[(int) ((long) sampleRate * millis / 1000) * channels * 2]; + } + + /** + * @return the arrays one after another. + */ + public static byte[] concat(byte[]... parts) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + for (byte[] part : parts) { + bytes.writeBytes(part); + } + + return bytes.toByteArray(); + } + + private static short[] load() { + try (InputStream input = SpeechFixture.class.getResourceAsStream("/audio/vad-speech-16k.pcm")) { + if (input == null) { + throw new IllegalStateException("The speech fixture is missing"); + } + + ByteBuffer pcm = ByteBuffer.wrap(input.readAllBytes()).order(ByteOrder.LITTLE_ENDIAN); + short[] samples = new short[pcm.remaining() / 2]; + + pcm.asShortBuffer().get(samples); + + return samples; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java new file mode 100644 index 000000000..03b65370c --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java @@ -0,0 +1,336 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import static dev.onvoid.webrtc.media.audio.SpeechFixture.silence; +import static dev.onvoid.webrtc.media.audio.SpeechFixture.speech; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.TestBase; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Speech detected in native code, in what a track's source hands to sinks + * and in what a factory's audio processing sends, with only the changes + * reaching the listener. + */ +class VoiceActivityListenerTest extends TestBase { + + /** Long enough to bridge every pause of the fixture. */ + private static final VoiceActivityConfig ONE_SEGMENT = new VoiceActivityConfig(0.5, Duration.ofMillis(500)); + + private final List changes = new CopyOnWriteArrayList<>(); + + private final VoiceActivityListener listener = changes::add; + + private CustomAudioSource source; + + private AudioTrack track; + + + @BeforeEach + void createTrack() { + // TestBase keeps one instance for all tests. + changes.clear(); + + source = new CustomAudioSource(); + track = factory.createAudioTrack("pushed", source); + } + + @AfterEach + void disposeTrack() { + track.dispose(); + source.dispose(); + } + + @Test + void pushedSpeechBeginsAndEnds() { + track.addVoiceActivityListener(ONE_SEGMENT, listener); + + push(speech(16000, 1), 16000, 1); + + assertEquals(List.of(true), changes); + + push(silence(1000, 16000, 1), 16000, 1); + + assertEquals(List.of(true, false), changes); + } + + @Test + void pushedSpeechAtAnyRate() { + // The detector takes 16 kHz mono; the rest is mixed and resampled. + int[][] formats = { { 8000, 1 }, { 44100, 2 }, { 48000, 1 }, { 48000, 2 } }; + + for (int[] format : formats) { + List heard = new CopyOnWriteArrayList<>(); + VoiceActivityListener formatListener = heard::add; + + track.addVoiceActivityListener(ONE_SEGMENT, formatListener); + + push(speech(format[0], format[1]), format[0], format[1]); + push(silence(1000, format[0], format[1]), format[0], format[1]); + + track.removeVoiceActivityListener(formatListener); + + assertEquals(List.of(true, false), heard, () -> format[0] + " Hz, " + format[1] + " channels"); + } + } + + @Test + void hangoverHoldsSpeech() { + // Both bridge the pauses of the fixture; the longer one ends the + // speech 500 ms later, give or take the 30 ms the detector rates. + int shorter = chunksUntilEnd(new VoiceActivityConfig(0.5, Duration.ofMillis(500))); + int longer = chunksUntilEnd(new VoiceActivityConfig(0.5, Duration.ofMillis(1000))); + + assertTrue(Math.abs(longer - shorter - 50) <= 3, () -> "Ended after " + shorter + " and " + longer + + " chunks"); + } + + @Test + void shortHangoverSplitsSpeech() { + // Without a hangover, the pauses between words end speech. + track.addVoiceActivityListener(new VoiceActivityConfig(0.5, Duration.ZERO), listener); + + push(speech(16000, 1), 16000, 1); + push(silence(100, 16000, 1), 16000, 1); + + assertAlternates(changes); + assertTrue(changes.size() > 2, () -> "Expected the speech split, got " + changes); + } + + @Test + void thresholdAboveSpeech() { + // The detector rates no audio at 1. + track.addVoiceActivityListener(new VoiceActivityConfig(1, Duration.ZERO), listener); + + push(speech(16000, 1), 16000, 1); + + assertEquals(List.of(), changes); + } + + @Test + void silenceIsNoSpeech() { + // Not even the detector's initial probability of 1 counts. + track.addVoiceActivityListener(new VoiceActivityConfig(0.02, Duration.ZERO), listener); + + push(silence(500, 16000, 1), 16000, 1); + + assertEquals(List.of(), changes); + } + + @Test + void eachListenerWithItsConfig() { + List other = new CopyOnWriteArrayList<>(); + + track.addVoiceActivityListener(ONE_SEGMENT, listener); + track.addVoiceActivityListener(new VoiceActivityConfig(1, Duration.ZERO), other::add); + + push(speech(16000, 1), 16000, 1); + push(silence(1000, 16000, 1), 16000, 1); + + assertEquals(List.of(true, false), changes); + assertEquals(List.of(), other); + } + + @Test + void removedListenerHearsNothing() { + track.addVoiceActivityListener(ONE_SEGMENT, listener); + // Adding it again keeps the one. + track.addVoiceActivityListener(ONE_SEGMENT, listener); + track.removeVoiceActivityListener(listener); + track.removeVoiceActivityListener(listener); + + push(speech(16000, 1), 16000, 1); + + assertEquals(List.of(), changes); + } + + @Test + void disposeRemovesListeners() { + AudioTrack other = factory.createAudioTrack("other", source); + + other.addVoiceActivityListener(ONE_SEGMENT, listener); + other.dispose(); + + push(speech(16000, 1), 16000, 1); + + assertEquals(List.of(), changes); + } + + @Test + void rejectsNull() { + assertThrows(NullPointerException.class, () -> track.addVoiceActivityListener(null, listener)); + assertThrows(NullPointerException.class, () -> track.addVoiceActivityListener(ONE_SEGMENT, null)); + assertThrows(NullPointerException.class, () -> track.removeVoiceActivityListener(null)); + } + + @Test + void factoryHearsItsProcessing() { + AudioProcessing processing = new AudioProcessing(); + HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); + PeerConnectionFactory processed = new PeerConnectionFactory(module, processing); + + try { + processed.addVoiceActivityListener(ONE_SEGMENT, listener); + + // What the factory's audio processing processes, as it would the + // microphone's audio. + process(processing, speech(48000, 1), 48000); + process(processing, silence(1000, 48000, 1), 48000); + + assertEquals(List.of(true, false), changes); + + processed.removeVoiceActivityListener(listener); + process(processing, speech(48000, 1), 48000); + + assertEquals(List.of(true, false), changes); + } + finally { + processed.dispose(); + module.dispose(); + processing.dispose(); + } + } + + @Test + void factoryDisposeRemovesListeners() { + // The application's audio processing outlives the factory. + AudioProcessing processing = new AudioProcessing(); + HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); + PeerConnectionFactory processed = new PeerConnectionFactory(module, processing); + + try { + processed.addVoiceActivityListener(ONE_SEGMENT, listener); + } + finally { + processed.dispose(); + module.dispose(); + } + + try { + process(processing, speech(16000, 1), 16000); + + assertEquals(List.of(), changes); + } + finally { + processing.dispose(); + } + } + + @Test + void factoryWithOwnProcessing() { + HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); + PeerConnectionFactory own = new PeerConnectionFactory(module); + + try { + own.addVoiceActivityListener(ONE_SEGMENT, listener); + own.removeVoiceActivityListener(listener); + own.addVoiceActivityListener(ONE_SEGMENT, listener); + } + finally { + // Removes the listener still added. + own.dispose(); + module.dispose(); + } + + assertThrows(NullPointerException.class, () -> own.addVoiceActivityListener(ONE_SEGMENT, listener)); + } + + @Test + void configDefaults() { + VoiceActivityConfig config = new VoiceActivityConfig(); + + assertEquals(VoiceActivityConfig.DEFAULT_THRESHOLD, config.getThreshold()); + assertEquals(VoiceActivityConfig.DEFAULT_HANGOVER, config.getHangover()); + } + + @Test + void configRejectsOutOfRange() { + for (double threshold : new double[] { 0, -0.5, 1.01, Double.NaN }) { + assertThrows(IllegalArgumentException.class, () -> new VoiceActivityConfig(threshold, Duration.ZERO)); + } + + assertThrows(IllegalArgumentException.class, () -> new VoiceActivityConfig(0.5, Duration.ofMillis(-10))); + assertThrows(IllegalArgumentException.class, () -> new VoiceActivityConfig(0.5, Duration.ofDays(30))); + assertThrows(NullPointerException.class, () -> new VoiceActivityConfig(0.5, null)); + } + + /** + * @return the 10 ms chunks of speech and silence pushed until the speech + * ended. + */ + private int chunksUntilEnd(VoiceActivityConfig config) { + List heard = new CopyOnWriteArrayList<>(); + VoiceActivityListener counting = heard::add; + byte[] pcm = SpeechFixture.concat(speech(16000, 1), silence(2000, 16000, 1)); + int chunks = 0; + + track.addVoiceActivityListener(config, counting); + + try { + for (int offset = 0; offset < pcm.length && heard.size() < 2; offset += 320) { + source.pushAudio(Arrays.copyOfRange(pcm, offset, offset + 320), 16, 16000, 1, 160); + chunks++; + } + } + finally { + track.removeVoiceActivityListener(counting); + } + + assertEquals(List.of(true, false), heard); + + return chunks; + } + + private void push(byte[] pcm, int sampleRate, int channels) { + int frames = sampleRate / 100; + int chunk = frames * channels * 2; + + for (int offset = 0; offset + chunk <= pcm.length; offset += chunk) { + source.pushAudio(Arrays.copyOfRange(pcm, offset, offset + chunk), 16, sampleRate, channels, frames); + } + } + + private static void process(AudioProcessing processing, byte[] pcm, int sampleRate) { + AudioProcessingStreamConfig format = new AudioProcessingStreamConfig(sampleRate, 1); + int chunk = sampleRate / 100 * 2; + byte[] output = new byte[chunk]; + + for (int offset = 0; offset + chunk <= pcm.length; offset += chunk) { + assertEquals(0, processing.processStream(Arrays.copyOfRange(pcm, offset, offset + chunk), format, + format, output)); + } + } + + private static void assertAlternates(List changes) { + for (int i = 0; i < changes.size(); i++) { + assertEquals(i % 2 == 0, changes.get(i), () -> "Expected alternating changes, got " + changes); + } + } +}