fork/rust-sdks
git clone https://git.julia.blue/fork/rust-sdks
a1b926e
livekit/livekit-ffi@0.13.0
1/* 2* Copyright 2023 LiveKit 3* 4* Licensed under the Apache License, Version 2.0 (the “License”); 5* you may not use this file except in compliance with the License. 6* You may obtain a copy of the License at 7* 8* http://www.apache.org/licenses/LICENSE-2.0 9* 10* Unless required by applicable law or agreed to in writing, software 11* distributed under the License is distributed on an “AS IS” BASIS, 12* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13* See the License for the specific language governing permissions and 14* limitations under the License. 15*/ 16 17#include "livekit/audio_track.h" 18 19#include <algorithm> 20#include <iostream> 21#include <iterator> 22#include <memory> 23 24#include "api/audio_options.h" 25#include "api/media_stream_interface.h" 26#include "api/task_queue/task_queue_base.h" 27#include "audio/remix_resample.h" 28#include "common_audio/include/audio_util.h" 29#include "livekit/global_task_queue.h" 30#include "rtc_base/checks.h" 31#include "rtc_base/logging.h" 32#include "rtc_base/ref_counted_object.h" 33#include "rtc_base/synchronization/mutex.h" 34#include "rtc_base/time_utils.h" 35#include "rust/cxx.h" 36#include "webrtc-sys/src/audio_track.rs.h" 37 38namespace livekit { 39 40inline cricket::AudioOptions to_native_audio_options ( 41const AudioSourceOptions & options ) { 42 cricket::AudioOptions rtc_options {}; 43rtc_options .echo_cancellation = options .echo_cancellation ; 44rtc_options .noise_suppression = options .noise_suppression ; 45rtc_options .auto_gain_control = options .auto_gain_control ; 46return rtc_options ; 47} 48 49inline AudioSourceOptions to_rust_audio_options ( 50const cricket::AudioOptions & rtc_options ) { 51AudioSourceOptions options {}; 52options .echo_cancellation = rtc_options .echo_cancellation .value_or (false); 53options .noise_suppression = rtc_options .noise_suppression .value_or (false); 54options .auto_gain_control = rtc_options .auto_gain_control .value_or (false); 55return options ; 56} 57 58AudioTrack ::AudioTrack (std::shared_ptr < RtcRuntime > rtc_runtime , 59 rtc::scoped_refptr < webrtc::AudioTrackInterface > track ) 60 :MediaStreamTrack (rtc_runtime , std::move (track )) {} 61 62AudioTrack ::~AudioTrack () { 63 webrtc::MutexLock lock (& mutex_ ); 64for (auto & sink :sinks_ ) { 65track ()-> RemoveSink (sink .get ()); 66 } 67} 68 69void AudioTrack ::add_sink (const std::shared_ptr < NativeAudioSink >& sink )const { 70 webrtc::MutexLock lock (& mutex_ ); 71track ()-> AddSink (sink .get ()); 72sinks_ .push_back (sink ); 73} 74 75void AudioTrack ::remove_sink ( 76const std::shared_ptr < NativeAudioSink >& sink )const { 77 webrtc::MutexLock lock (& mutex_ ); 78track ()-> RemoveSink (sink .get ()); 79sinks_ .erase (std::remove (sinks_ .begin (),sinks_ .end (),sink ),sinks_ .end ()); 80} 81 82NativeAudioSink ::NativeAudioSink (rust::Box < AudioSinkWrapper > observer , 83int sample_rate , 84int num_channels ) 85 :observer_ (std::move (observer )), 86sample_rate_ (sample_rate ), 87num_channels_ (num_channels ) { 88frame_ .sample_rate_hz_ = sample_rate ; 89frame_ .num_channels_ = num_channels ; 90} 91 92void NativeAudioSink ::OnData (const void * audio_data , 93int bits_per_sample , 94int sample_rate , 95size_t number_of_channels , 96size_t number_of_frames ) { 97RTC_CHECK_EQ (16 ,bits_per_sample ); 98 99const int16_t * data = static_cast < const int16_t *> (audio_data ); 100 101if (sample_rate_ != sample_rate || num_channels_ != number_of_channels ) { 102// resample/remix before capturing 103 webrtc::voe::RemixAndResample (data ,number_of_frames ,number_of_channels , 104sample_rate ,& resampler_ ,& frame_ ); 105 106 rust::Slice < const int16_t > rust_slice ( 107frame_ .data (),frame_ .num_channels ()* frame_ .samples_per_channel ()); 108 109observer_ -> on_data (rust_slice ,frame_ .sample_rate_hz (), 110frame_ .num_channels (),frame_ .samples_per_channel ()); 111 112 }else { 113 rust::Slice < const int16_t > rust_slice ( 114data ,number_of_channels * number_of_frames ); 115 116observer_ -> on_data (rust_slice ,sample_rate ,number_of_channels , 117number_of_frames ); 118 } 119} 120 121std::shared_ptr < NativeAudioSink > new_native_audio_sink ( 122 rust::Box < AudioSinkWrapper > observer , 123int sample_rate , 124int num_channels ) { 125return std::make_shared < NativeAudioSink > (std::move (observer ),sample_rate , 126num_channels ); 127} 128 129AudioTrackSource ::InternalSource ::InternalSource ( 130const cricket::AudioOptions & options , 131int sample_rate , 132int num_channels , 133int queue_size_ms ,// must be a multiple of 10ms 134 webrtc::TaskQueueFactory * task_queue_factory ) 135 :sample_rate_ (sample_rate ), 136num_channels_ (num_channels ), 137capture_userdata_ (nullptr ), 138on_complete_ (nullptr ) { 139if (!queue_size_ms ) 140return ;// no audio queue 141 142// start sending silence when there is nothing on the queue for 10 frames 143// (100ms) 144const int silence_frames_threshold = 10 ; 145missed_frames_ = silence_frames_threshold ; 146 147int samples10ms = sample_rate /100 * num_channels ; 148 149silence_buffer_ = new int16_t [samples10ms ](); 150queue_size_samples_ = queue_size_ms /10 * samples10ms ; 151notify_threshold_samples_ = queue_size_samples_ ;// TODO: this is currently 152// using x2 the queue size 153buffer_ .reserve (queue_size_samples_ + notify_threshold_samples_ ); 154 155audio_queue_ = 156task_queue_factory -> CreateTaskQueue ( 157"AudioSourceCapture" , webrtc::TaskQueueFactory ::Priority ::NORMAL ); 158 159audio_task_ = webrtc::RepeatingTaskHandle ::Start ( 160audio_queue_ .get (), 161 [this ,samples10ms ]() { 162 webrtc::MutexLock lock (& mutex_ ); 163 164if (buffer_ .size () >=samples10ms ) { 165for (auto sink :sinks_ ) 166sink -> OnData (buffer_ .data (),sizeof (int16_t )* 8 ,sample_rate_ , 167num_channels_ ,samples10ms /num_channels_ ); 168 169buffer_ .erase (buffer_ .begin (),buffer_ .begin ()+ samples10ms ); 170 }else { 171missed_frames_ ++ ; 172if (missed_frames_ >=silence_frames_threshold ) { 173for (auto sink :sinks_ ) 174sink -> OnData (silence_buffer_ ,sizeof (int16_t )* 8 ,sample_rate_ , 175num_channels_ ,samples10ms /num_channels_ ); 176 } 177 } 178 179if (on_complete_ && buffer_ .size () <=notify_threshold_samples_ ) { 180on_complete_ (capture_userdata_ ); 181on_complete_ = nullptr ; 182capture_userdata_ = nullptr ; 183 } 184 185return webrtc::TimeDelta ::Millis (10 ); 186 }, 187 webrtc::TaskQueueBase ::DelayPrecision ::kHigh ); 188} 189 190AudioTrackSource ::InternalSource ::~InternalSource () { 191delete []silence_buffer_ ; 192} 193 194bool AudioTrackSource ::InternalSource ::capture_frame ( 195 rust::Slice < const int16_t > data , 196uint32_t sample_rate , 197uint32_t number_of_channels , 198size_t number_of_frames , 199const SourceContext * ctx , 200void (* on_complete )(const SourceContext * )) { 201 webrtc::MutexLock lock (& mutex_ ); 202 203if (queue_size_samples_ ) { 204int available = 205 (queue_size_samples_ + notify_threshold_samples_ )- buffer_ .size (); 206if (available < data .size ()) 207return false; 208 209if (on_complete_ || capture_userdata_ ) 210return false; 211 212buffer_ .insert (buffer_ .end (),data .begin (),data .end ()); 213 214if (buffer_ .size () <=notify_threshold_samples_ ) { 215on_complete (ctx );// complete directly 216 }else { 217on_complete_ = on_complete ; 218capture_userdata_ = ctx ; 219 } 220 221 }else { 222// capture directly when the queue buffer is 0 (frame size must be 10ms) 223for (auto sink :sinks_ ) 224sink -> OnData (data .data (),sizeof (int16_t )* 8 ,sample_rate , 225number_of_channels ,number_of_frames ); 226 } 227 228return true; 229} 230 231void AudioTrackSource ::InternalSource ::clear_buffer () { 232 webrtc::MutexLock lock (& mutex_ ); 233buffer_ .clear (); 234} 235 236webrtc::MediaSourceInterface ::SourceState 237AudioTrackSource ::InternalSource ::state ()const { 238return webrtc::MediaSourceInterface ::SourceState ::kLive ; 239} 240 241bool AudioTrackSource ::InternalSource ::remote ()const { 242return false; 243} 244 245const cricket::AudioOptions AudioTrackSource ::InternalSource ::options ()const { 246 webrtc::MutexLock lock (& mutex_ ); 247return options_ ; 248} 249 250void AudioTrackSource ::InternalSource ::set_options ( 251const cricket::AudioOptions & options ) { 252 webrtc::MutexLock lock (& mutex_ ); 253options_ = options ; 254} 255 256void AudioTrackSource ::InternalSource ::AddSink ( 257 webrtc::AudioTrackSinkInterface * sink ) { 258 webrtc::MutexLock lock (& mutex_ ); 259sinks_ .push_back (sink ); 260} 261 262void AudioTrackSource ::InternalSource ::RemoveSink ( 263 webrtc::AudioTrackSinkInterface * sink ) { 264 webrtc::MutexLock lock (& mutex_ ); 265sinks_ .erase (std::remove (sinks_ .begin (),sinks_ .end (),sink ),sinks_ .end ()); 266} 267 268AudioTrackSource ::AudioTrackSource (AudioSourceOptions options , 269int sample_rate , 270int num_channels , 271int queue_size_ms , 272 webrtc::TaskQueueFactory * task_queue_factory ) 273 :source_ (rtc::make_ref_counted < InternalSource > ( 274to_native_audio_options (options ), 275sample_rate , 276num_channels , 277queue_size_ms , 278task_queue_factory )) {} 279 280AudioSourceOptions AudioTrackSource ::audio_options ()const { 281return to_rust_audio_options (source_ -> options ()); 282} 283 284void AudioTrackSource ::set_audio_options ( 285const AudioSourceOptions & options )const { 286source_ -> set_options (to_native_audio_options (options )); 287} 288 289bool AudioTrackSource ::capture_frame ( 290 rust::Slice < const int16_t > audio_data , 291uint32_t sample_rate , 292uint32_t number_of_channels , 293size_t number_of_frames , 294const SourceContext * ctx , 295void (* on_complete )(const SourceContext * ))const { 296return source_ -> capture_frame (audio_data ,sample_rate ,number_of_channels , 297number_of_frames ,ctx ,on_complete ); 298} 299 300void AudioTrackSource ::clear_buffer ()const { 301source_ -> clear_buffer (); 302} 303 304std::shared_ptr < AudioTrackSource > new_audio_track_source ( 305AudioSourceOptions options , 306int sample_rate , 307int num_channels , 308int queue_size_ms ) { 309return std::make_shared < AudioTrackSource > (options ,sample_rate ,num_channels , 310queue_size_ms , 311GetGlobalTaskQueueFactory ()); 312} 313 314rtc::scoped_refptr < AudioTrackSource ::InternalSource > AudioTrackSource ::get () 315const { 316return source_ ; 317} 318 319}// namespace livekit