fork/rust-sdks

git clone https://git.julia.blue/fork/rust-sdks

aoife cassidychore(webrtc-sys): bump to m125_release (#505)a1b926e

livekit/livekit-ffi@0.13.0
10.3 KiB319 linesraw
1/*
2 * Copyright 2023 LiveKit
3 *
4 * Licensed under the Apache License, Version 2.0 (the “License”);
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an “AS IS” BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "livekit/audio_track.h"
18
19#include <algorithm>
20#include <iostream>
21#include <iterator>
22#include <memory>
23
24#include "api/audio_options.h"
25#include "api/media_stream_interface.h"
26#include "api/task_queue/task_queue_base.h"
27#include "audio/remix_resample.h"
28#include "common_audio/include/audio_util.h"
29#include "livekit/global_task_queue.h"
30#include "rtc_base/checks.h"
31#include "rtc_base/logging.h"
32#include "rtc_base/ref_counted_object.h"
33#include "rtc_base/synchronization/mutex.h"
34#include "rtc_base/time_utils.h"
35#include "rust/cxx.h"
36#include "webrtc-sys/src/audio_track.rs.h"
37
38namespace livekit {
39
40inline cricket::AudioOptions to_native_audio_options(
41    const AudioSourceOptions& options) {
42  cricket::AudioOptions rtc_options{};
43  rtc_options.echo_cancellation = options.echo_cancellation;
44  rtc_options.noise_suppression = options.noise_suppression;
45  rtc_options.auto_gain_control = options.auto_gain_control;
46  return rtc_options;
47}
48
49inline AudioSourceOptions to_rust_audio_options(
50    const cricket::AudioOptions& rtc_options) {
51  AudioSourceOptions options{};
52  options.echo_cancellation = rtc_options.echo_cancellation.value_or(false);
53  options.noise_suppression = rtc_options.noise_suppression.value_or(false);
54  options.auto_gain_control = rtc_options.auto_gain_control.value_or(false);
55  return options;
56}
57
58AudioTrack::AudioTrack(std::shared_ptr<RtcRuntime> rtc_runtime,
59                       rtc::scoped_refptr<webrtc::AudioTrackInterface> track)
60    : MediaStreamTrack(rtc_runtime, std::move(track)) {}
61
62AudioTrack::~AudioTrack() {
63  webrtc::MutexLock lock(&mutex_);
64  for (auto& sink : sinks_) {
65    track()->RemoveSink(sink.get());
66  }
67}
68
69void AudioTrack::add_sink(const std::shared_ptr<NativeAudioSink>& sink) const {
70  webrtc::MutexLock lock(&mutex_);
71  track()->AddSink(sink.get());
72  sinks_.push_back(sink);
73}
74
75void AudioTrack::remove_sink(
76    const std::shared_ptr<NativeAudioSink>& sink) const {
77  webrtc::MutexLock lock(&mutex_);
78  track()->RemoveSink(sink.get());
79  sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end());
80}
81
82NativeAudioSink::NativeAudioSink(rust::Box<AudioSinkWrapper> observer,
83                                 int sample_rate,
84                                 int num_channels)
85    : observer_(std::move(observer)),
86      sample_rate_(sample_rate),
87      num_channels_(num_channels) {
88  frame_.sample_rate_hz_ = sample_rate;
89  frame_.num_channels_ = num_channels;
90}
91
92void NativeAudioSink::OnData(const void* audio_data,
93                             int bits_per_sample,
94                             int sample_rate,
95                             size_t number_of_channels,
96                             size_t number_of_frames) {
97  RTC_CHECK_EQ(16, bits_per_sample);
98
99  const int16_t* data = static_cast<const int16_t*>(audio_data);
100
101  if (sample_rate_ != sample_rate || num_channels_ != number_of_channels) {
102    // resample/remix before capturing
103    webrtc::voe::RemixAndResample(data, number_of_frames, number_of_channels,
104                                  sample_rate, &resampler_, &frame_);
105
106    rust::Slice<const int16_t> rust_slice(
107        frame_.data(), frame_.num_channels() * frame_.samples_per_channel());
108
109    observer_->on_data(rust_slice, frame_.sample_rate_hz(),
110                       frame_.num_channels(), frame_.samples_per_channel());
111
112  } else {
113    rust::Slice<const int16_t> rust_slice(
114        data, number_of_channels * number_of_frames);
115
116    observer_->on_data(rust_slice, sample_rate, number_of_channels,
117                       number_of_frames);
118  }
119}
120
121std::shared_ptr<NativeAudioSink> new_native_audio_sink(
122    rust::Box<AudioSinkWrapper> observer,
123    int sample_rate,
124    int num_channels) {
125  return std::make_shared<NativeAudioSink>(std::move(observer), sample_rate,
126                                           num_channels);
127}
128
129AudioTrackSource::InternalSource::InternalSource(
130    const cricket::AudioOptions& options,
131    int sample_rate,
132    int num_channels,
133    int queue_size_ms,  // must be a multiple of 10ms
134    webrtc::TaskQueueFactory* task_queue_factory)
135    : sample_rate_(sample_rate),
136      num_channels_(num_channels),
137      capture_userdata_(nullptr),
138      on_complete_(nullptr) {
139  if (!queue_size_ms)
140    return;  // no audio queue
141
142  // start sending silence when there is nothing on the queue for 10 frames
143  // (100ms)
144  const int silence_frames_threshold = 10;
145  missed_frames_ = silence_frames_threshold;
146
147  int samples10ms = sample_rate / 100 * num_channels;
148
149  silence_buffer_ = new int16_t[samples10ms]();
150  queue_size_samples_ = queue_size_ms / 10 * samples10ms;
151  notify_threshold_samples_ = queue_size_samples_;  // TODO: this is currently
152                                                    // using x2 the queue size
153  buffer_.reserve(queue_size_samples_ + notify_threshold_samples_);
154
155  audio_queue_ =
156      task_queue_factory->CreateTaskQueue(
157          "AudioSourceCapture", webrtc::TaskQueueFactory::Priority::NORMAL);
158
159  audio_task_ = webrtc::RepeatingTaskHandle::Start(
160      audio_queue_.get(),
161      [this, samples10ms]() {
162        webrtc::MutexLock lock(&mutex_);
163
164        if (buffer_.size() >= samples10ms) {
165          for (auto sink : sinks_)
166            sink->OnData(buffer_.data(), sizeof(int16_t) * 8, sample_rate_,
167                         num_channels_, samples10ms / num_channels_);
168
169          buffer_.erase(buffer_.begin(), buffer_.begin() + samples10ms);
170        } else {
171          missed_frames_++;
172          if (missed_frames_ >= silence_frames_threshold) {
173            for (auto sink : sinks_)
174              sink->OnData(silence_buffer_, sizeof(int16_t) * 8, sample_rate_,
175                           num_channels_, samples10ms / num_channels_);
176          }
177        }
178
179        if (on_complete_ && buffer_.size() <= notify_threshold_samples_) {
180          on_complete_(capture_userdata_);
181          on_complete_ = nullptr;
182          capture_userdata_ = nullptr;
183        }
184
185        return webrtc::TimeDelta::Millis(10);
186      },
187      webrtc::TaskQueueBase::DelayPrecision::kHigh);
188}
189
190AudioTrackSource::InternalSource::~InternalSource() {
191  delete[] silence_buffer_;
192}
193
194bool AudioTrackSource::InternalSource::capture_frame(
195    rust::Slice<const int16_t> data,
196    uint32_t sample_rate,
197    uint32_t number_of_channels,
198    size_t number_of_frames,
199    const SourceContext* ctx,
200    void (*on_complete)(const SourceContext*)) {
201  webrtc::MutexLock lock(&mutex_);
202
203  if (queue_size_samples_) {
204    int available =
205        (queue_size_samples_ + notify_threshold_samples_) - buffer_.size();
206    if (available < data.size())
207      return false;
208
209    if (on_complete_ || capture_userdata_)
210      return false;
211
212    buffer_.insert(buffer_.end(), data.begin(), data.end());
213
214    if (buffer_.size() <= notify_threshold_samples_) {
215      on_complete(ctx);  // complete directly
216    } else {
217      on_complete_ = on_complete;
218      capture_userdata_ = ctx;
219    }
220
221  } else {
222    // capture directly when the queue buffer is 0 (frame size must be 10ms)
223    for (auto sink : sinks_)
224      sink->OnData(data.data(), sizeof(int16_t) * 8, sample_rate,
225                   number_of_channels, number_of_frames);
226  }
227
228  return true;
229}
230
231void AudioTrackSource::InternalSource::clear_buffer() {
232  webrtc::MutexLock lock(&mutex_);
233  buffer_.clear();
234}
235
236webrtc::MediaSourceInterface::SourceState
237AudioTrackSource::InternalSource::state() const {
238  return webrtc::MediaSourceInterface::SourceState::kLive;
239}
240
241bool AudioTrackSource::InternalSource::remote() const {
242  return false;
243}
244
245const cricket::AudioOptions AudioTrackSource::InternalSource::options() const {
246  webrtc::MutexLock lock(&mutex_);
247  return options_;
248}
249
250void AudioTrackSource::InternalSource::set_options(
251    const cricket::AudioOptions& options) {
252  webrtc::MutexLock lock(&mutex_);
253  options_ = options;
254}
255
256void AudioTrackSource::InternalSource::AddSink(
257    webrtc::AudioTrackSinkInterface* sink) {
258  webrtc::MutexLock lock(&mutex_);
259  sinks_.push_back(sink);
260}
261
262void AudioTrackSource::InternalSource::RemoveSink(
263    webrtc::AudioTrackSinkInterface* sink) {
264  webrtc::MutexLock lock(&mutex_);
265  sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end());
266}
267
268AudioTrackSource::AudioTrackSource(AudioSourceOptions options,
269                                   int sample_rate,
270                                   int num_channels,
271                                   int queue_size_ms,
272                                   webrtc::TaskQueueFactory* task_queue_factory)
273    : source_(rtc::make_ref_counted<InternalSource>(
274          to_native_audio_options(options),
275          sample_rate,
276          num_channels,
277          queue_size_ms,
278          task_queue_factory)) {}
279
280AudioSourceOptions AudioTrackSource::audio_options() const {
281  return to_rust_audio_options(source_->options());
282}
283
284void AudioTrackSource::set_audio_options(
285    const AudioSourceOptions& options) const {
286  source_->set_options(to_native_audio_options(options));
287}
288
289bool AudioTrackSource::capture_frame(
290    rust::Slice<const int16_t> audio_data,
291    uint32_t sample_rate,
292    uint32_t number_of_channels,
293    size_t number_of_frames,
294    const SourceContext* ctx,
295    void (*on_complete)(const SourceContext*)) const {
296  return source_->capture_frame(audio_data, sample_rate, number_of_channels,
297                                number_of_frames, ctx, on_complete);
298}
299
300void AudioTrackSource::clear_buffer() const {
301  source_->clear_buffer();
302}
303
304std::shared_ptr<AudioTrackSource> new_audio_track_source(
305    AudioSourceOptions options,
306    int sample_rate,
307    int num_channels,
308    int queue_size_ms) {
309  return std::make_shared<AudioTrackSource>(options, sample_rate, num_channels,
310                                            queue_size_ms,
311                                            GetGlobalTaskQueueFactory());
312}
313
314rtc::scoped_refptr<AudioTrackSource::InternalSource> AudioTrackSource::get()
315    const {
316  return source_;
317}
318
319}  // namespace livekit