fork/rust-sdks

git clone https://git.julia.blue/fork/rust-sdks

Théo Monnomexpose apm stream_delay (#616)7cbdddf

main
10.8 KiB287 linesraw
1// Copyright 2023 LiveKit, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15syntax = "proto2";
16
17package livekit.proto;
18option csharp_namespace = "LiveKit.Proto";
19
20// import "handle.proto";
21import "e2ee.proto";
22import "track.proto";
23import "track_publication.proto";
24import "room.proto";
25import "video_frame.proto";
26import "audio_frame.proto";
27import "rpc.proto";
28
29// **How is the livekit-ffi working:
30// We refer as the ffi server the Rust server that is running the LiveKit client implementation, and we
31// refer as the ffi client the foreign language that commumicates with the ffi server. (e.g Python SDK, Unity SDK, etc...)
32//
33// We expose the Rust client implementation of livekit using the protocol defined here.
34// Everything starts with a FfiRequest, which is a oneof message that contains all the possible
35// requests that can be made to the ffi server.
36// The server will then respond with a FfiResponse, which is also a oneof message that contains
37// all the possible responses.
38// The first request sent to the server must be an InitializeRequest, which contains the a pointer
39// to the callback function that will be used to send events and async responses to the ffi client.
40// (e.g participant joined, track published, etc...)
41//
42// **Useful things know when collaborating on the protocol:**
43// Everything is subject to discussion and change :-)
44//
45// - The ffi client implementation must never forget to correctly dispose all the owned handles
46//   that it receives from the server.
47//
48// Therefore, the ffi client is easier to implement if there is less handles to manage.
49//
50// - We are mainly using FfiHandle on info messages (e.g: RoomInfo, TrackInfo, etc...)
51//   For this reason, info are only sent once, at creation (We're not using them for updates, we can infer them from
52//   events on the client implementation).
53//   e.g: set speaking to true when we receive a ActiveSpeakerChanged event.
54
55// This is the input of livekit_ffi_request function
56// We always expect a response (FFIResponse, even if it's empty)
57message FfiRequest {
58  oneof message {
59    DisposeRequest dispose = 2;
60
61    // Room
62    ConnectRequest connect = 3;
63    DisconnectRequest disconnect = 4;
64    PublishTrackRequest publish_track = 5;
65    UnpublishTrackRequest unpublish_track = 6;
66    PublishDataRequest publish_data = 7;
67    SetSubscribedRequest set_subscribed = 8;
68    SetLocalMetadataRequest set_local_metadata = 9;
69    SetLocalNameRequest set_local_name = 10;
70    SetLocalAttributesRequest set_local_attributes = 11;
71    GetSessionStatsRequest get_session_stats = 12;
72    PublishTranscriptionRequest publish_transcription = 13;
73    PublishSipDtmfRequest publish_sip_dtmf = 14;
74
75    // Track
76    CreateVideoTrackRequest create_video_track = 15;
77    CreateAudioTrackRequest create_audio_track = 16;
78    LocalTrackMuteRequest local_track_mute = 17;
79    EnableRemoteTrackRequest enable_remote_track = 18;
80    GetStatsRequest get_stats = 19;
81    SetTrackSubscriptionPermissionsRequest set_track_subscription_permissions = 48;
82
83    // Video
84    NewVideoStreamRequest new_video_stream = 20;
85    NewVideoSourceRequest new_video_source = 21;
86    CaptureVideoFrameRequest capture_video_frame = 22;
87    VideoConvertRequest video_convert = 23;
88    VideoStreamFromParticipantRequest video_stream_from_participant = 24;
89
90    // Audio
91    NewAudioStreamRequest new_audio_stream = 25;
92    NewAudioSourceRequest new_audio_source = 26;
93    CaptureAudioFrameRequest capture_audio_frame = 27;
94    ClearAudioBufferRequest clear_audio_buffer = 28;
95    NewAudioResamplerRequest new_audio_resampler = 29;
96    RemixAndResampleRequest remix_and_resample = 30;
97    E2eeRequest e2ee = 31;
98    AudioStreamFromParticipantRequest audio_stream_from_participant = 32;
99    NewSoxResamplerRequest new_sox_resampler = 33;
100    PushSoxResamplerRequest push_sox_resampler = 34;
101    FlushSoxResamplerRequest flush_sox_resampler = 35;
102    SendChatMessageRequest send_chat_message = 36;
103    EditChatMessageRequest edit_chat_message = 37;
104
105    // RPC
106    PerformRpcRequest perform_rpc = 38;
107    RegisterRpcMethodRequest register_rpc_method = 39;
108    UnregisterRpcMethodRequest unregister_rpc_method = 40;
109    RpcMethodInvocationResponseRequest rpc_method_invocation_response = 41;
110
111    // Track Publication
112    EnableRemoteTrackPublicationRequest enable_remote_track_publication = 42;
113    UpdateRemoteTrackPublicationDimensionRequest update_remote_track_publication_dimension = 43;
114
115    // Data Streams
116    SendStreamHeaderRequest send_stream_header = 44;
117    SendStreamChunkRequest send_stream_chunk = 45;
118    SendStreamTrailerRequest send_stream_trailer = 46;
119
120    // Data Channel
121    SetDataChannelBufferedAmountLowThresholdRequest set_data_channel_buffered_amount_low_threshold = 47;
122
123    // Audio Filter Plugin
124    LoadAudioFilterPluginRequest load_audio_filter_plugin = 49;
125
126    NewApmRequest new_apm = 50;
127    ApmProcessStreamRequest apm_process_stream = 51;
128    ApmProcessReverseStreamRequest apm_process_reverse_stream = 52;
129    ApmSetStreamDelayRequest apm_set_stream_delay = 53;
130
131    // NEXT_ID: 54
132  }
133}
134
135// This is the output of livekit_ffi_request function.
136message FfiResponse {
137  oneof message {
138    DisposeResponse dispose = 2;
139
140    // Room
141    ConnectResponse connect = 3;
142    DisconnectResponse disconnect = 4;
143    PublishTrackResponse publish_track = 5;
144    UnpublishTrackResponse unpublish_track = 6;
145    PublishDataResponse publish_data = 7;
146    SetSubscribedResponse set_subscribed = 8;
147    SetLocalMetadataResponse set_local_metadata = 9;
148    SetLocalNameResponse set_local_name = 10;
149    SetLocalAttributesResponse set_local_attributes = 11;
150    GetSessionStatsResponse get_session_stats = 12;
151    PublishTranscriptionResponse publish_transcription = 13;
152    PublishSipDtmfResponse publish_sip_dtmf = 14;
153
154    // Track
155    CreateVideoTrackResponse create_video_track = 15;
156    CreateAudioTrackResponse create_audio_track = 16;
157    LocalTrackMuteResponse local_track_mute = 17;
158    EnableRemoteTrackResponse enable_remote_track = 18;
159    GetStatsResponse get_stats = 19;
160    SetTrackSubscriptionPermissionsResponse set_track_subscription_permissions = 47;
161
162    // Video
163    NewVideoStreamResponse new_video_stream = 20;
164    NewVideoSourceResponse new_video_source = 21;
165    CaptureVideoFrameResponse capture_video_frame = 22;
166    VideoConvertResponse video_convert = 23;
167    VideoStreamFromParticipantResponse video_stream_from_participant = 24;
168
169    // Audio
170    NewAudioStreamResponse new_audio_stream = 25;
171    NewAudioSourceResponse new_audio_source = 26;
172    CaptureAudioFrameResponse capture_audio_frame = 27;
173    ClearAudioBufferResponse clear_audio_buffer = 28;
174    NewAudioResamplerResponse new_audio_resampler = 29;
175    RemixAndResampleResponse remix_and_resample = 30;
176    AudioStreamFromParticipantResponse audio_stream_from_participant = 31;
177    E2eeResponse e2ee = 32;
178    NewSoxResamplerResponse new_sox_resampler = 33;
179    PushSoxResamplerResponse push_sox_resampler = 34;
180    FlushSoxResamplerResponse flush_sox_resampler = 35;
181    SendChatMessageResponse send_chat_message = 36;
182    // RPC
183    PerformRpcResponse perform_rpc = 37;
184    RegisterRpcMethodResponse register_rpc_method = 38;
185    UnregisterRpcMethodResponse unregister_rpc_method = 39;
186    RpcMethodInvocationResponseResponse rpc_method_invocation_response = 40;
187    // Track Publication
188    EnableRemoteTrackPublicationResponse enable_remote_track_publication = 41;
189    UpdateRemoteTrackPublicationDimensionResponse update_remote_track_publication_dimension = 42;
190
191    // Data Streams
192    SendStreamHeaderResponse send_stream_header = 43;
193    SendStreamChunkResponse send_stream_chunk = 44;
194    SendStreamTrailerResponse send_stream_trailer = 45;
195
196    // Data Channel
197    SetDataChannelBufferedAmountLowThresholdResponse set_data_channel_buffered_amount_low_threshold = 46;
198
199    // Audio Filter Plugin
200    LoadAudioFilterPluginResponse load_audio_filter_plugin = 48;
201
202    NewApmResponse new_apm = 49;
203    ApmProcessStreamResponse apm_process_stream = 50;
204    ApmProcessReverseStreamResponse apm_process_reverse_stream = 51;
205    ApmSetStreamDelayResponse apm_set_stream_delay = 52;
206
207    // NEXT_ID: 53
208  }
209}
210
211// To minimize complexity, participant events are not included in the protocol.
212// It is easily deducible from the room events and it turned out that is is easier to implement
213// on the ffi client side.
214message FfiEvent {
215  oneof message {
216    RoomEvent room_event = 1;
217    TrackEvent track_event = 2;
218    VideoStreamEvent video_stream_event = 3;
219    AudioStreamEvent audio_stream_event = 4;
220    ConnectCallback connect = 5;
221    DisconnectCallback disconnect = 7;
222    DisposeCallback dispose = 8;
223    PublishTrackCallback publish_track = 9;
224    UnpublishTrackCallback unpublish_track = 10;
225    PublishDataCallback publish_data = 11;
226    PublishTranscriptionCallback publish_transcription = 12;
227    CaptureAudioFrameCallback capture_audio_frame = 13;
228    SetLocalMetadataCallback set_local_metadata = 14;
229    SetLocalNameCallback set_local_name = 15;
230    SetLocalAttributesCallback set_local_attributes = 16;
231    GetStatsCallback get_stats = 17;
232    LogBatch logs = 18;
233    GetSessionStatsCallback get_session_stats = 19;
234    Panic panic = 20;
235    PublishSipDtmfCallback publish_sip_dtmf = 21;
236    SendChatMessageCallback chat_message = 22;
237    PerformRpcCallback perform_rpc = 23;
238    RpcMethodInvocationEvent rpc_method_invocation = 24;
239    SendStreamHeaderCallback send_stream_header = 25;
240    SendStreamChunkCallback send_stream_chunk = 26;
241    SendStreamTrailerCallback send_stream_trailer = 27;
242  }
243}
244
245// Stop all rooms synchronously (Do we need async here?).
246// e.g: This is used for the Unity Editor after each assemblies reload.
247// TODO(theomonnom): Implement a debug mode where we can find all leaked handles?
248message DisposeRequest {
249  required bool async = 1;
250}
251message DisposeResponse {
252  optional uint64 async_id = 1; // None if sync
253}
254
255message DisposeCallback {
256  required uint64 async_id = 1;
257}
258
259enum LogLevel {
260  LOG_ERROR = 0;
261  LOG_WARN = 1;
262  LOG_INFO = 2;
263  LOG_DEBUG = 3;
264  LOG_TRACE = 4;
265}
266
267message LogRecord {
268  required LogLevel level = 1;
269  required string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc...
270  optional string module_path = 3;
271  optional string file = 4;
272  optional uint32 line = 5;
273  required string message = 6;
274}
275
276message LogBatch {
277  repeated LogRecord records = 1;
278}
279
280message Panic {
281  required string message = 1;
282}
283
284// TODO(theomonnom): Debug messages (Print handles).
285
286
287