fork/rust-sdks

git clone https://git.julia.blue/fork/rust-sdks

Théo Monnomadd AudioProcessingModule (#580)7e51858

livekit/livekit-ffi@0.13.0
10.7 KiB285 linesraw
1// Copyright 2023 LiveKit, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15syntax = "proto2";
16
17package livekit.proto;
18option csharp_namespace = "LiveKit.Proto";
19
20// import "handle.proto";
21import "e2ee.proto";
22import "track.proto";
23import "track_publication.proto";
24import "room.proto";
25import "video_frame.proto";
26import "audio_frame.proto";
27import "rpc.proto";
28
29// **How is the livekit-ffi working:
30// We refer as the ffi server the Rust server that is running the LiveKit client implementation, and we
31// refer as the ffi client the foreign language that commumicates with the ffi server. (e.g Python SDK, Unity SDK, etc...)
32//
33// We expose the Rust client implementation of livekit using the protocol defined here.
34// Everything starts with a FfiRequest, which is a oneof message that contains all the possible
35// requests that can be made to the ffi server.
36// The server will then respond with a FfiResponse, which is also a oneof message that contains
37// all the possible responses.
38// The first request sent to the server must be an InitializeRequest, which contains the a pointer
39// to the callback function that will be used to send events and async responses to the ffi client.
40// (e.g participant joined, track published, etc...)
41//
42// **Useful things know when collaborating on the protocol:**
43// Everything is subject to discussion and change :-)
44//
45// - The ffi client implementation must never forget to correctly dispose all the owned handles
46//   that it receives from the server.
47//
48// Therefore, the ffi client is easier to implement if there is less handles to manage.
49//
50// - We are mainly using FfiHandle on info messages (e.g: RoomInfo, TrackInfo, etc...)
51//   For this reason, info are only sent once, at creation (We're not using them for updates, we can infer them from
52//   events on the client implementation).
53//   e.g: set speaking to true when we receive a ActiveSpeakerChanged event.
54
55// This is the input of livekit_ffi_request function
56// We always expect a response (FFIResponse, even if it's empty)
57message FfiRequest {
58  oneof message {
59    DisposeRequest dispose = 2;
60
61    // Room
62    ConnectRequest connect = 3;
63    DisconnectRequest disconnect = 4;
64    PublishTrackRequest publish_track = 5;
65    UnpublishTrackRequest unpublish_track = 6;
66    PublishDataRequest publish_data = 7;
67    SetSubscribedRequest set_subscribed = 8;
68    SetLocalMetadataRequest set_local_metadata = 9;
69    SetLocalNameRequest set_local_name = 10;
70    SetLocalAttributesRequest set_local_attributes = 11;
71    GetSessionStatsRequest get_session_stats = 12;
72    PublishTranscriptionRequest publish_transcription = 13;
73    PublishSipDtmfRequest publish_sip_dtmf = 14;
74
75    // Track
76    CreateVideoTrackRequest create_video_track = 15;
77    CreateAudioTrackRequest create_audio_track = 16;
78    LocalTrackMuteRequest local_track_mute = 17;
79    EnableRemoteTrackRequest enable_remote_track = 18;
80    GetStatsRequest get_stats = 19;
81    SetTrackSubscriptionPermissionsRequest set_track_subscription_permissions = 48;
82
83    // Video
84    NewVideoStreamRequest new_video_stream = 20;
85    NewVideoSourceRequest new_video_source = 21;
86    CaptureVideoFrameRequest capture_video_frame = 22;
87    VideoConvertRequest video_convert = 23;
88    VideoStreamFromParticipantRequest video_stream_from_participant = 24;
89
90    // Audio
91    NewAudioStreamRequest new_audio_stream = 25;
92    NewAudioSourceRequest new_audio_source = 26;
93    CaptureAudioFrameRequest capture_audio_frame = 27;
94    ClearAudioBufferRequest clear_audio_buffer = 28;
95    NewAudioResamplerRequest new_audio_resampler = 29;
96    RemixAndResampleRequest remix_and_resample = 30;
97    E2eeRequest e2ee = 31;
98    AudioStreamFromParticipantRequest audio_stream_from_participant = 32;
99    NewSoxResamplerRequest new_sox_resampler = 33;
100    PushSoxResamplerRequest push_sox_resampler = 34;
101    FlushSoxResamplerRequest flush_sox_resampler = 35;
102    SendChatMessageRequest send_chat_message = 36;
103    EditChatMessageRequest edit_chat_message = 37;
104
105    // RPC
106    PerformRpcRequest perform_rpc = 38;
107    RegisterRpcMethodRequest register_rpc_method = 39;
108    UnregisterRpcMethodRequest unregister_rpc_method = 40;
109    RpcMethodInvocationResponseRequest rpc_method_invocation_response = 41;
110
111    // Track Publication
112    EnableRemoteTrackPublicationRequest enable_remote_track_publication = 42;
113    UpdateRemoteTrackPublicationDimensionRequest update_remote_track_publication_dimension = 43;
114
115    // Data Streams
116    SendStreamHeaderRequest send_stream_header = 44;
117    SendStreamChunkRequest send_stream_chunk = 45;
118    SendStreamTrailerRequest send_stream_trailer = 46;
119
120    // Data Channel
121    SetDataChannelBufferedAmountLowThresholdRequest set_data_channel_buffered_amount_low_threshold = 47;
122
123    // Audio Filter Plugin
124    LoadAudioFilterPluginRequest load_audio_filter_plugin = 49;
125
126    NewApmRequest new_apm = 50;
127    ApmProcessStreamRequest apm_process_stream = 51;
128    ApmProcessReverseStreamRequest apm_process_reverse_stream = 52;
129
130    // NEXT_ID: 53
131  }
132}
133
134// This is the output of livekit_ffi_request function.
135message FfiResponse {
136  oneof message {
137    DisposeResponse dispose = 2;
138
139    // Room
140    ConnectResponse connect = 3;
141    DisconnectResponse disconnect = 4;
142    PublishTrackResponse publish_track = 5;
143    UnpublishTrackResponse unpublish_track = 6;
144    PublishDataResponse publish_data = 7;
145    SetSubscribedResponse set_subscribed = 8;
146    SetLocalMetadataResponse set_local_metadata = 9;
147    SetLocalNameResponse set_local_name = 10;
148    SetLocalAttributesResponse set_local_attributes = 11;
149    GetSessionStatsResponse get_session_stats = 12;
150    PublishTranscriptionResponse publish_transcription = 13;
151    PublishSipDtmfResponse publish_sip_dtmf = 14;
152
153    // Track
154    CreateVideoTrackResponse create_video_track = 15;
155    CreateAudioTrackResponse create_audio_track = 16;
156    LocalTrackMuteResponse local_track_mute = 17;
157    EnableRemoteTrackResponse enable_remote_track = 18;
158    GetStatsResponse get_stats = 19;
159    SetTrackSubscriptionPermissionsResponse set_track_subscription_permissions = 47;
160
161    // Video
162    NewVideoStreamResponse new_video_stream = 20;
163    NewVideoSourceResponse new_video_source = 21;
164    CaptureVideoFrameResponse capture_video_frame = 22;
165    VideoConvertResponse video_convert = 23;
166    VideoStreamFromParticipantResponse video_stream_from_participant = 24;
167
168    // Audio
169    NewAudioStreamResponse new_audio_stream = 25;
170    NewAudioSourceResponse new_audio_source = 26;
171    CaptureAudioFrameResponse capture_audio_frame = 27;
172    ClearAudioBufferResponse clear_audio_buffer = 28;
173    NewAudioResamplerResponse new_audio_resampler = 29;
174    RemixAndResampleResponse remix_and_resample = 30;
175    AudioStreamFromParticipantResponse audio_stream_from_participant = 31;
176    E2eeResponse e2ee = 32;
177    NewSoxResamplerResponse new_sox_resampler = 33;
178    PushSoxResamplerResponse push_sox_resampler = 34;
179    FlushSoxResamplerResponse flush_sox_resampler = 35;
180    SendChatMessageResponse send_chat_message = 36;
181    // RPC
182    PerformRpcResponse perform_rpc = 37;
183    RegisterRpcMethodResponse register_rpc_method = 38;
184    UnregisterRpcMethodResponse unregister_rpc_method = 39;
185    RpcMethodInvocationResponseResponse rpc_method_invocation_response = 40;
186    // Track Publication
187    EnableRemoteTrackPublicationResponse enable_remote_track_publication = 41;
188    UpdateRemoteTrackPublicationDimensionResponse update_remote_track_publication_dimension = 42;
189
190    // Data Streams
191    SendStreamHeaderResponse send_stream_header = 43;
192    SendStreamChunkResponse send_stream_chunk = 44;
193    SendStreamTrailerResponse send_stream_trailer = 45;
194
195    // Data Channel
196    SetDataChannelBufferedAmountLowThresholdResponse set_data_channel_buffered_amount_low_threshold = 46;
197
198    // Audio Filter Plugin
199    LoadAudioFilterPluginResponse load_audio_filter_plugin = 48;
200
201    NewApmResponse new_apm = 49;
202    ApmProcessStreamResponse apm_process_stream = 50;
203    ApmProcessReverseStreamResponse apm_process_reverse_stream = 51;
204
205    // NEXT_ID: 52
206  }
207}
208
209// To minimize complexity, participant events are not included in the protocol.
210// It is easily deducible from the room events and it turned out that is is easier to implement
211// on the ffi client side.
212message FfiEvent {
213  oneof message {
214    RoomEvent room_event = 1;
215    TrackEvent track_event = 2;
216    VideoStreamEvent video_stream_event = 3;
217    AudioStreamEvent audio_stream_event = 4;
218    ConnectCallback connect = 5;
219    DisconnectCallback disconnect = 7;
220    DisposeCallback dispose = 8;
221    PublishTrackCallback publish_track = 9;
222    UnpublishTrackCallback unpublish_track = 10;
223    PublishDataCallback publish_data = 11;
224    PublishTranscriptionCallback publish_transcription = 12;
225    CaptureAudioFrameCallback capture_audio_frame = 13;
226    SetLocalMetadataCallback set_local_metadata = 14;
227    SetLocalNameCallback set_local_name = 15;
228    SetLocalAttributesCallback set_local_attributes = 16;
229    GetStatsCallback get_stats = 17;
230    LogBatch logs = 18;
231    GetSessionStatsCallback get_session_stats = 19;
232    Panic panic = 20;
233    PublishSipDtmfCallback publish_sip_dtmf = 21;
234    SendChatMessageCallback chat_message = 22;
235    PerformRpcCallback perform_rpc = 23;
236    RpcMethodInvocationEvent rpc_method_invocation = 24;
237    SendStreamHeaderCallback send_stream_header = 25;
238    SendStreamChunkCallback send_stream_chunk = 26;
239    SendStreamTrailerCallback send_stream_trailer = 27;
240  }
241}
242
243// Stop all rooms synchronously (Do we need async here?).
244// e.g: This is used for the Unity Editor after each assemblies reload.
245// TODO(theomonnom): Implement a debug mode where we can find all leaked handles?
246message DisposeRequest {
247  required bool async = 1;
248}
249message DisposeResponse {
250  optional uint64 async_id = 1; // None if sync
251}
252
253message DisposeCallback {
254  required uint64 async_id = 1;
255}
256
257enum LogLevel {
258  LOG_ERROR = 0;
259  LOG_WARN = 1;
260  LOG_INFO = 2;
261  LOG_DEBUG = 3;
262  LOG_TRACE = 4;
263}
264
265message LogRecord {
266  required LogLevel level = 1;
267  required string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc...
268  optional string module_path = 3;
269  optional string file = 4;
270  optional uint32 line = 5;
271  required string message = 6;
272}
273
274message LogBatch {
275  repeated LogRecord records = 1;
276}
277
278message Panic {
279  required string message = 1;
280}
281
282// TODO(theomonnom): Debug messages (Print handles).
283
284
285