fork/rust-sdks

git clone https://git.julia.blue/fork/rust-sdks

Théo Monnomfeat: add ffi datachannel & mute events (#88)25f4c9a

livekit/livekit-ffi@0.13.0
6.3 KiB202 linesraw
1/*
2 * Copyright 2023 LiveKit
3 *
4 * Licensed under the Apache License, Version 2.0 (the “License”);
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an “AS IS” BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "livekit/video_track.h"
18
19#include <algorithm>
20#include <iostream>
21#include <memory>
22
23#include "api/media_stream_interface.h"
24#include "api/video/video_frame.h"
25#include "api/video/video_rotation.h"
26#include "audio/remix_resample.h"
27#include "common_audio/include/audio_util.h"
28#include "livekit/media_stream.h"
29#include "livekit/video_track.h"
30#include "rtc_base/logging.h"
31#include "rtc_base/ref_counted_object.h"
32#include "rtc_base/synchronization/mutex.h"
33#include "rtc_base/time_utils.h"
34#include "webrtc-sys/src/video_track.rs.h"
35
36namespace livekit {
37
38VideoTrack::VideoTrack(std::shared_ptr<RtcRuntime> rtc_runtime,
39                       rtc::scoped_refptr<webrtc::VideoTrackInterface> track)
40    : MediaStreamTrack(rtc_runtime, std::move(track)) {}
41
42VideoTrack::~VideoTrack() {
43  webrtc::MutexLock lock(&mutex_);
44  for (auto& sink : sinks_) {
45    track()->RemoveSink(sink.get());
46  }
47}
48
49void VideoTrack::add_sink(const std::shared_ptr<NativeVideoSink>& sink) const {
50  webrtc::MutexLock lock(&mutex_);
51  track()->AddOrUpdateSink(sink.get(),
52                           rtc::VideoSinkWants());  // TODO(theomonnom): Expose
53                                                    // VideoSinkWants to Rust?
54  sinks_.push_back(sink);
55}
56
57void VideoTrack::remove_sink(
58    const std::shared_ptr<NativeVideoSink>& sink) const {
59  webrtc::MutexLock lock(&mutex_);
60  track()->RemoveSink(sink.get());
61  sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end());
62}
63
64void VideoTrack::set_should_receive(bool should_receive) const {
65  track()->set_should_receive(should_receive);
66}
67
68bool VideoTrack::should_receive() const {
69  return track()->should_receive();
70}
71
72ContentHint VideoTrack::content_hint() const {
73  return static_cast<ContentHint>(track()->content_hint());
74}
75
76void VideoTrack::set_content_hint(ContentHint hint) const {
77  track()->set_content_hint(
78      static_cast<webrtc::VideoTrackInterface::ContentHint>(hint));
79}
80
81NativeVideoSink::NativeVideoSink(rust::Box<VideoSinkWrapper> observer)
82    : observer_(std::move(observer)) {}
83
84void NativeVideoSink::OnFrame(const webrtc::VideoFrame& frame) {
85  observer_->on_frame(std::make_unique<VideoFrame>(frame));
86}
87
88void NativeVideoSink::OnDiscardedFrame() {
89  observer_->on_discarded_frame();
90}
91
92void NativeVideoSink::OnConstraintsChanged(
93    const webrtc::VideoTrackSourceConstraints& constraints) {
94  VideoTrackSourceConstraints cst;
95  cst.has_min_fps = constraints.min_fps.has_value();
96  cst.min_fps = constraints.min_fps.value_or(0);
97  cst.has_max_fps = constraints.max_fps.has_value();
98  cst.max_fps = constraints.max_fps.value_or(0);
99  observer_->on_constraints_changed(cst);
100}
101
102std::shared_ptr<NativeVideoSink> new_native_video_sink(
103    rust::Box<VideoSinkWrapper> observer) {
104  return std::make_shared<NativeVideoSink>(std::move(observer));
105}
106
107VideoTrackSource::InternalSource::InternalSource(
108    const VideoResolution& resolution)
109    : rtc::AdaptedVideoTrackSource(4), resolution_(resolution) {}
110
111VideoTrackSource::InternalSource::~InternalSource() {}
112
113bool VideoTrackSource::InternalSource::is_screencast() const {
114  return false;
115}
116
117absl::optional<bool> VideoTrackSource::InternalSource::needs_denoising() const {
118  return false;
119}
120
121webrtc::MediaSourceInterface::SourceState
122VideoTrackSource::InternalSource::state() const {
123  return SourceState::kLive;
124}
125
126bool VideoTrackSource::InternalSource::remote() const {
127  return false;
128}
129
130VideoResolution VideoTrackSource::InternalSource::video_resolution() const {
131  webrtc::MutexLock lock(&mutex_);
132  return resolution_;
133}
134
135bool VideoTrackSource::InternalSource::on_captured_frame(
136    const webrtc::VideoFrame& frame) {
137  webrtc::MutexLock lock(&mutex_);
138
139  int64_t aligned_timestamp_us = timestamp_aligner_.TranslateTimestamp(
140      frame.timestamp_us(), rtc::TimeMicros());
141
142  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
143      frame.video_frame_buffer();
144
145  if (resolution_.height == 0 || resolution_.width == 0) {
146    resolution_ = VideoResolution{static_cast<uint32_t>(buffer->width()),
147                                  static_cast<uint32_t>(buffer->height())};
148  }
149
150  int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y;
151  if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us,
152                  &adapted_width, &adapted_height, &crop_width, &crop_height,
153                  &crop_x, &crop_y)) {
154    return false;
155  }
156
157  if (adapted_width != frame.width() || adapted_height != frame.height()) {
158    buffer = buffer->CropAndScale(crop_x, crop_y, crop_width, crop_height,
159                                  adapted_width, adapted_height);
160  }
161
162  webrtc::VideoRotation rotation = frame.rotation();
163  if (apply_rotation() && rotation != webrtc::kVideoRotation_0) {
164    // If the buffer is I420, rtc::AdaptedVideoTrackSource will handle the
165    // rotation for us.
166    buffer = buffer->ToI420();
167  }
168
169  OnFrame(webrtc::VideoFrame::Builder()
170              .set_video_frame_buffer(buffer)
171              .set_rotation(rotation)
172              .set_timestamp_us(aligned_timestamp_us)
173              .build());
174
175  return true;
176}
177
178VideoTrackSource::VideoTrackSource(const VideoResolution& resolution) {
179  source_ = rtc::make_ref_counted<InternalSource>(resolution);
180}
181
182VideoResolution VideoTrackSource::video_resolution() const {
183  return source_->video_resolution();
184}
185
186bool VideoTrackSource::on_captured_frame(
187    const std::unique_ptr<VideoFrame>& frame) const {
188  auto rtc_frame = frame->get();
189  return source_->on_captured_frame(rtc_frame);
190}
191
192rtc::scoped_refptr<VideoTrackSource::InternalSource> VideoTrackSource::get()
193    const {
194  return source_;
195}
196
197std::shared_ptr<VideoTrackSource> new_video_track_source(
198    const VideoResolution& resolution) {
199  return std::make_shared<VideoTrackSource>(resolution);
200}
201
202}  // namespace livekit