fork/rust-sdks

git clone https://git.julia.blue/fork/rust-sdks

Théo Monnomadd AudioProcessingModule (#580)7e51858

livekit/livekit-ffi@0.13.0
7.3 KiB309 linesraw
1// Copyright 2023 LiveKit, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15syntax = "proto2";
16
17package livekit.proto;
18option csharp_namespace = "LiveKit.Proto";
19
20import "handle.proto";
21import "track.proto";
22
23// Create a new AudioStream
24// AudioStream is used to receive audio frames from a track
25message NewAudioStreamRequest {
26  required uint64 track_handle = 1;
27  required AudioStreamType type = 2;
28  optional uint32 sample_rate = 3;
29  optional uint32 num_channels = 4;
30  optional string audio_filter_module_id = 5; // Unique identifier passed in LoadAudioFilterPluginRequest
31  optional string audio_filter_options = 6;
32}
33message NewAudioStreamResponse { required OwnedAudioStream stream = 1; }
34
35message AudioStreamFromParticipantRequest {
36  required uint64 participant_handle = 1;
37  required AudioStreamType type = 2;
38  optional TrackSource track_source = 3;
39  optional uint32 sample_rate = 5;
40  optional uint32 num_channels = 6;
41  optional string audio_filter_module_id = 7;
42  optional string audio_filter_options = 8;
43}
44
45message AudioStreamFromParticipantResponse { required OwnedAudioStream stream = 1; }
46
47// Create a new AudioSource
48message NewAudioSourceRequest {
49  required AudioSourceType type = 1;
50  optional AudioSourceOptions options = 2;
51  required uint32 sample_rate = 3;
52  required uint32 num_channels = 4;
53  optional uint32 queue_size_ms = 5;
54}
55message NewAudioSourceResponse { required OwnedAudioSource source = 1; }
56
57// Push a frame to an AudioSource 
58// The data provided must be available as long as the client receive the callback.
59message CaptureAudioFrameRequest { 
60  required uint64 source_handle = 1;
61  required AudioFrameBufferInfo buffer = 2;
62}
63message CaptureAudioFrameResponse {
64  required uint64 async_id = 1;
65}
66message CaptureAudioFrameCallback {
67  required uint64 async_id = 1;
68  optional string error = 2;
69}
70
71message ClearAudioBufferRequest {
72  required uint64 source_handle = 1;
73}
74message ClearAudioBufferResponse {}
75
76// Create a new AudioResampler
77message NewAudioResamplerRequest {}
78message NewAudioResamplerResponse {
79  required OwnedAudioResampler resampler = 1;
80}
81
82// Remix and resample an audio frame
83message RemixAndResampleRequest {
84  required uint64 resampler_handle = 1;
85  required AudioFrameBufferInfo buffer = 2;
86  required uint32 num_channels = 3;
87  required uint32 sample_rate = 4;
88}
89
90message RemixAndResampleResponse {
91  required OwnedAudioFrameBuffer buffer = 1;
92}
93
94// AEC
95
96
97message NewApmRequest {
98  required bool echo_canceller_enabled = 1;
99  required bool gain_controller_enabled = 2;
100  required bool high_pass_filter_enabled = 3;
101  required bool noise_suppression_enabled = 4;
102}
103message NewApmResponse {
104  required OwnedApm apm = 1;
105}
106
107message ApmProcessStreamRequest {
108  required uint64 apm_handle = 1;
109  required uint64 data_ptr = 2; // *mut i16
110  required uint32 size = 3; // in bytes
111  required uint32 sample_rate = 4;
112  required uint32 num_channels = 5;
113}
114
115message ApmProcessStreamResponse {
116  optional string error = 1;
117}
118
119message ApmProcessReverseStreamRequest {
120  required uint64 apm_handle = 1;
121  required uint64 data_ptr = 2; // *mut i16
122  required uint32 size = 3; // in bytes
123  required uint32 sample_rate = 4;
124  required uint32 num_channels = 5;
125}
126
127message ApmProcessReverseStreamResponse {
128  optional string error = 1;
129}
130
131// New resampler using SoX (much better quality)
132
133message NewSoxResamplerRequest {
134  required double input_rate = 1;
135  required double output_rate  = 2;
136  required uint32 num_channels = 3;
137  required SoxResamplerDataType input_data_type = 4;
138  required SoxResamplerDataType output_data_type = 5;
139  required SoxQualityRecipe quality_recipe = 6;
140  optional uint32 flags = 7;
141}
142message NewSoxResamplerResponse {
143  oneof message {
144    OwnedSoxResampler resampler = 1;
145    string error = 2;
146  }
147  
148}
149
150message PushSoxResamplerRequest {
151  required uint64 resampler_handle = 1;
152  required uint64 data_ptr = 2; // *const i16
153  required uint32 size = 3; // in bytes
154}
155
156message PushSoxResamplerResponse {
157  required uint64 output_ptr = 1; // *const i16 (could be null)
158  required uint32 size = 2; // in bytes
159  optional string error = 3;
160}
161
162message FlushSoxResamplerRequest {
163  required uint64 resampler_handle = 1;
164}
165
166message FlushSoxResamplerResponse {
167  required uint64 output_ptr = 1; // *const i16 (could be null)
168  required uint32 size = 2; // in bytes
169  optional string error = 3;
170}
171
172enum SoxResamplerDataType {
173  // TODO(theomonnom): support other datatypes (shouldn't really be needed)
174  SOXR_DATATYPE_INT16I = 0;
175  SOXR_DATATYPE_INT16S = 1;
176}
177
178enum SoxQualityRecipe {
179  SOXR_QUALITY_QUICK = 0;
180  SOXR_QUALITY_LOW = 1;
181  SOXR_QUALITY_MEDIUM = 2;
182  SOXR_QUALITY_HIGH = 3;
183  SOXR_QUALITY_VERYHIGH = 4;
184}
185
186enum SoxFlagBits {
187  SOXR_ROLLOFF_SMALL = 0;  // 1 << 0
188  SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1
189  SOXR_ROLLOFF_NONE = 2;   // 1 << 2
190  SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3
191  SOXR_DOUBLE_PRECISION = 4; // 1 << 4
192  SOXR_VR = 5; // 1 << 5
193}
194
195
196
197//
198// AudioFrame buffer
199//
200
201message AudioFrameBufferInfo {
202  required uint64 data_ptr = 1; // *const i16
203  required uint32 num_channels = 2;
204  required uint32 sample_rate = 3;
205  required uint32 samples_per_channel = 4;
206}
207
208message OwnedAudioFrameBuffer {
209  required FfiOwnedHandle handle = 1;
210  required AudioFrameBufferInfo info = 2;
211}
212
213//
214// AudioStream
215//
216
217enum AudioStreamType {
218  AUDIO_STREAM_NATIVE = 0;
219  AUDIO_STREAM_HTML = 1;
220}
221
222message AudioStreamInfo {
223  required AudioStreamType type = 1;
224}
225
226message OwnedAudioStream {
227  required FfiOwnedHandle handle = 1;
228  required AudioStreamInfo info = 2;
229}
230
231message AudioStreamEvent {
232  required uint64 stream_handle = 1;
233  oneof message { 
234    AudioFrameReceived frame_received = 2;
235    AudioStreamEOS eos = 3;
236  }
237}
238
239message AudioFrameReceived {
240  required OwnedAudioFrameBuffer frame = 1;
241}
242
243message AudioStreamEOS {}
244
245//
246// AudioSource
247//
248
249message AudioSourceOptions {
250  required bool echo_cancellation = 1;
251  required bool noise_suppression = 2;
252  required bool auto_gain_control = 3;
253}
254
255enum AudioSourceType {
256  AUDIO_SOURCE_NATIVE = 0;
257}
258
259message AudioSourceInfo {
260  required AudioSourceType type = 2;
261}
262
263message OwnedAudioSource {
264  required FfiOwnedHandle handle = 1;
265  required AudioSourceInfo info = 2;
266}
267
268//
269// AudioResampler
270//
271
272message AudioResamplerInfo { }
273
274message OwnedAudioResampler {
275  required FfiOwnedHandle handle = 1;
276  required AudioResamplerInfo info = 2;
277}
278
279
280//
281// AEC
282//
283
284message OwnedApm {
285  required FfiOwnedHandle handle = 1;
286}
287
288//
289// Sox AudioResampler
290//
291
292
293message SoxResamplerInfo {}
294
295message OwnedSoxResampler {
296  required FfiOwnedHandle handle = 1;
297  required SoxResamplerInfo info = 2;
298}
299
300// Audio Filter Plugin
301message LoadAudioFilterPluginRequest {
302  required string plugin_path = 1;  // path for ffi audio filter plugin
303  repeated string dependencies = 2; // Optional: paths for dependency dylibs
304  required string module_id = 3;    // Unique identifier of the plugin
305}
306
307message LoadAudioFilterPluginResponse {
308  optional string error = 1;
309}