fork/rust-sdks

git clone https://git.julia.blue/fork/rust-sdks

Théo Monnomexpose apm stream_delay (#616)7cbdddf

main
7.5 KiB319 linesraw
1// Copyright 2023 LiveKit, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15syntax = "proto2";
16
17package livekit.proto;
18option csharp_namespace = "LiveKit.Proto";
19
20import "handle.proto";
21import "track.proto";
22
23// Create a new AudioStream
24// AudioStream is used to receive audio frames from a track
25message NewAudioStreamRequest {
26  required uint64 track_handle = 1;
27  required AudioStreamType type = 2;
28  optional uint32 sample_rate = 3;
29  optional uint32 num_channels = 4;
30  optional string audio_filter_module_id = 5; // Unique identifier passed in LoadAudioFilterPluginRequest
31  optional string audio_filter_options = 6;
32}
33message NewAudioStreamResponse { required OwnedAudioStream stream = 1; }
34
35message AudioStreamFromParticipantRequest {
36  required uint64 participant_handle = 1;
37  required AudioStreamType type = 2;
38  optional TrackSource track_source = 3;
39  optional uint32 sample_rate = 5;
40  optional uint32 num_channels = 6;
41  optional string audio_filter_module_id = 7;
42  optional string audio_filter_options = 8;
43}
44
45message AudioStreamFromParticipantResponse { required OwnedAudioStream stream = 1; }
46
47// Create a new AudioSource
48message NewAudioSourceRequest {
49  required AudioSourceType type = 1;
50  optional AudioSourceOptions options = 2;
51  required uint32 sample_rate = 3;
52  required uint32 num_channels = 4;
53  optional uint32 queue_size_ms = 5;
54}
55message NewAudioSourceResponse { required OwnedAudioSource source = 1; }
56
57// Push a frame to an AudioSource 
58// The data provided must be available as long as the client receive the callback.
59message CaptureAudioFrameRequest { 
60  required uint64 source_handle = 1;
61  required AudioFrameBufferInfo buffer = 2;
62}
63message CaptureAudioFrameResponse {
64  required uint64 async_id = 1;
65}
66message CaptureAudioFrameCallback {
67  required uint64 async_id = 1;
68  optional string error = 2;
69}
70
71message ClearAudioBufferRequest {
72  required uint64 source_handle = 1;
73}
74message ClearAudioBufferResponse {}
75
76// Create a new AudioResampler
77message NewAudioResamplerRequest {}
78message NewAudioResamplerResponse {
79  required OwnedAudioResampler resampler = 1;
80}
81
82// Remix and resample an audio frame
83message RemixAndResampleRequest {
84  required uint64 resampler_handle = 1;
85  required AudioFrameBufferInfo buffer = 2;
86  required uint32 num_channels = 3;
87  required uint32 sample_rate = 4;
88}
89
90message RemixAndResampleResponse {
91  required OwnedAudioFrameBuffer buffer = 1;
92}
93
94// AEC
95
96
97message NewApmRequest {
98  required bool echo_canceller_enabled = 1;
99  required bool gain_controller_enabled = 2;
100  required bool high_pass_filter_enabled = 3;
101  required bool noise_suppression_enabled = 4;
102}
103message NewApmResponse {
104  required OwnedApm apm = 1;
105}
106
107message ApmProcessStreamRequest {
108  required uint64 apm_handle = 1;
109  required uint64 data_ptr = 2; // *mut i16
110  required uint32 size = 3; // in bytes
111  required uint32 sample_rate = 4;
112  required uint32 num_channels = 5;
113}
114
115message ApmProcessStreamResponse {
116  optional string error = 1;
117}
118
119message ApmProcessReverseStreamRequest {
120  required uint64 apm_handle = 1;
121  required uint64 data_ptr = 2; // *mut i16
122  required uint32 size = 3; // in bytes
123  required uint32 sample_rate = 4;
124  required uint32 num_channels = 5;
125}
126
127message ApmProcessReverseStreamResponse {
128  optional string error = 1;
129}
130
131message ApmSetStreamDelayRequest {
132  required uint64 apm_handle = 1;
133  required int32 delay_ms = 2;
134}
135
136message ApmSetStreamDelayResponse {
137  optional string error = 1;
138}
139
140
141// New resampler using SoX (much better quality)
142
143message NewSoxResamplerRequest {
144  required double input_rate = 1;
145  required double output_rate  = 2;
146  required uint32 num_channels = 3;
147  required SoxResamplerDataType input_data_type = 4;
148  required SoxResamplerDataType output_data_type = 5;
149  required SoxQualityRecipe quality_recipe = 6;
150  optional uint32 flags = 7;
151}
152message NewSoxResamplerResponse {
153  oneof message {
154    OwnedSoxResampler resampler = 1;
155    string error = 2;
156  }
157  
158}
159
160message PushSoxResamplerRequest {
161  required uint64 resampler_handle = 1;
162  required uint64 data_ptr = 2; // *const i16
163  required uint32 size = 3; // in bytes
164}
165
166message PushSoxResamplerResponse {
167  required uint64 output_ptr = 1; // *const i16 (could be null)
168  required uint32 size = 2; // in bytes
169  optional string error = 3;
170}
171
172message FlushSoxResamplerRequest {
173  required uint64 resampler_handle = 1;
174}
175
176message FlushSoxResamplerResponse {
177  required uint64 output_ptr = 1; // *const i16 (could be null)
178  required uint32 size = 2; // in bytes
179  optional string error = 3;
180}
181
182enum SoxResamplerDataType {
183  // TODO(theomonnom): support other datatypes (shouldn't really be needed)
184  SOXR_DATATYPE_INT16I = 0;
185  SOXR_DATATYPE_INT16S = 1;
186}
187
188enum SoxQualityRecipe {
189  SOXR_QUALITY_QUICK = 0;
190  SOXR_QUALITY_LOW = 1;
191  SOXR_QUALITY_MEDIUM = 2;
192  SOXR_QUALITY_HIGH = 3;
193  SOXR_QUALITY_VERYHIGH = 4;
194}
195
196enum SoxFlagBits {
197  SOXR_ROLLOFF_SMALL = 0;  // 1 << 0
198  SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1
199  SOXR_ROLLOFF_NONE = 2;   // 1 << 2
200  SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3
201  SOXR_DOUBLE_PRECISION = 4; // 1 << 4
202  SOXR_VR = 5; // 1 << 5
203}
204
205
206
207//
208// AudioFrame buffer
209//
210
211message AudioFrameBufferInfo {
212  required uint64 data_ptr = 1; // *const i16
213  required uint32 num_channels = 2;
214  required uint32 sample_rate = 3;
215  required uint32 samples_per_channel = 4;
216}
217
218message OwnedAudioFrameBuffer {
219  required FfiOwnedHandle handle = 1;
220  required AudioFrameBufferInfo info = 2;
221}
222
223//
224// AudioStream
225//
226
227enum AudioStreamType {
228  AUDIO_STREAM_NATIVE = 0;
229  AUDIO_STREAM_HTML = 1;
230}
231
232message AudioStreamInfo {
233  required AudioStreamType type = 1;
234}
235
236message OwnedAudioStream {
237  required FfiOwnedHandle handle = 1;
238  required AudioStreamInfo info = 2;
239}
240
241message AudioStreamEvent {
242  required uint64 stream_handle = 1;
243  oneof message { 
244    AudioFrameReceived frame_received = 2;
245    AudioStreamEOS eos = 3;
246  }
247}
248
249message AudioFrameReceived {
250  required OwnedAudioFrameBuffer frame = 1;
251}
252
253message AudioStreamEOS {}
254
255//
256// AudioSource
257//
258
259message AudioSourceOptions {
260  required bool echo_cancellation = 1;
261  required bool noise_suppression = 2;
262  required bool auto_gain_control = 3;
263}
264
265enum AudioSourceType {
266  AUDIO_SOURCE_NATIVE = 0;
267}
268
269message AudioSourceInfo {
270  required AudioSourceType type = 2;
271}
272
273message OwnedAudioSource {
274  required FfiOwnedHandle handle = 1;
275  required AudioSourceInfo info = 2;
276}
277
278//
279// AudioResampler
280//
281
282message AudioResamplerInfo { }
283
284message OwnedAudioResampler {
285  required FfiOwnedHandle handle = 1;
286  required AudioResamplerInfo info = 2;
287}
288
289
290//
291// AEC
292//
293
294message OwnedApm {
295  required FfiOwnedHandle handle = 1;
296}
297
298//
299// Sox AudioResampler
300//
301
302
303message SoxResamplerInfo {}
304
305message OwnedSoxResampler {
306  required FfiOwnedHandle handle = 1;
307  required SoxResamplerInfo info = 2;
308}
309
310// Audio Filter Plugin
311message LoadAudioFilterPluginRequest {
312  required string plugin_path = 1;  // path for ffi audio filter plugin
313  repeated string dependencies = 2; // Optional: paths for dependency dylibs
314  required string module_id = 3;    // Unique identifier of the plugin
315}
316
317message LoadAudioFilterPluginResponse {
318  optional string error = 1;
319}