// Copyright 2023 LiveKit, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; import "handle.proto"; import "track.proto"; // Create a new AudioStream // AudioStream is used to receive audio frames from a track message NewAudioStreamRequest { required uint64 track_handle = 1; required AudioStreamType type = 2; optional uint32 sample_rate = 3; optional uint32 num_channels = 4; optional string audio_filter_module_id = 5; // Unique identifier passed in LoadAudioFilterPluginRequest optional string audio_filter_options = 6; } message NewAudioStreamResponse { required OwnedAudioStream stream = 1; } message AudioStreamFromParticipantRequest { required uint64 participant_handle = 1; required AudioStreamType type = 2; optional TrackSource track_source = 3; optional uint32 sample_rate = 5; optional uint32 num_channels = 6; optional string audio_filter_module_id = 7; optional string audio_filter_options = 8; } message AudioStreamFromParticipantResponse { required OwnedAudioStream stream = 1; } // Create a new AudioSource message NewAudioSourceRequest { required AudioSourceType type = 1; optional AudioSourceOptions options = 2; required uint32 sample_rate = 3; required uint32 num_channels = 4; optional uint32 queue_size_ms = 5; } message NewAudioSourceResponse { required OwnedAudioSource source = 1; } // Push a frame to an AudioSource // The data provided must be available as long as the client receive the callback. message CaptureAudioFrameRequest { required uint64 source_handle = 1; required AudioFrameBufferInfo buffer = 2; } message CaptureAudioFrameResponse { required uint64 async_id = 1; } message CaptureAudioFrameCallback { required uint64 async_id = 1; optional string error = 2; } message ClearAudioBufferRequest { required uint64 source_handle = 1; } message ClearAudioBufferResponse {} // Create a new AudioResampler message NewAudioResamplerRequest {} message NewAudioResamplerResponse { required OwnedAudioResampler resampler = 1; } // Remix and resample an audio frame message RemixAndResampleRequest { required uint64 resampler_handle = 1; required AudioFrameBufferInfo buffer = 2; required uint32 num_channels = 3; required uint32 sample_rate = 4; } message RemixAndResampleResponse { required OwnedAudioFrameBuffer buffer = 1; } // AEC message NewApmRequest { required bool echo_canceller_enabled = 1; required bool gain_controller_enabled = 2; required bool high_pass_filter_enabled = 3; required bool noise_suppression_enabled = 4; } message NewApmResponse { required OwnedApm apm = 1; } message ApmProcessStreamRequest { required uint64 apm_handle = 1; required uint64 data_ptr = 2; // *mut i16 required uint32 size = 3; // in bytes required uint32 sample_rate = 4; required uint32 num_channels = 5; } message ApmProcessStreamResponse { optional string error = 1; } message ApmProcessReverseStreamRequest { required uint64 apm_handle = 1; required uint64 data_ptr = 2; // *mut i16 required uint32 size = 3; // in bytes required uint32 sample_rate = 4; required uint32 num_channels = 5; } message ApmProcessReverseStreamResponse { optional string error = 1; } // New resampler using SoX (much better quality) message NewSoxResamplerRequest { required double input_rate = 1; required double output_rate = 2; required uint32 num_channels = 3; required SoxResamplerDataType input_data_type = 4; required SoxResamplerDataType output_data_type = 5; required SoxQualityRecipe quality_recipe = 6; optional uint32 flags = 7; } message NewSoxResamplerResponse { oneof message { OwnedSoxResampler resampler = 1; string error = 2; } } message PushSoxResamplerRequest { required uint64 resampler_handle = 1; required uint64 data_ptr = 2; // *const i16 required uint32 size = 3; // in bytes } message PushSoxResamplerResponse { required uint64 output_ptr = 1; // *const i16 (could be null) required uint32 size = 2; // in bytes optional string error = 3; } message FlushSoxResamplerRequest { required uint64 resampler_handle = 1; } message FlushSoxResamplerResponse { required uint64 output_ptr = 1; // *const i16 (could be null) required uint32 size = 2; // in bytes optional string error = 3; } enum SoxResamplerDataType { // TODO(theomonnom): support other datatypes (shouldn't really be needed) SOXR_DATATYPE_INT16I = 0; SOXR_DATATYPE_INT16S = 1; } enum SoxQualityRecipe { SOXR_QUALITY_QUICK = 0; SOXR_QUALITY_LOW = 1; SOXR_QUALITY_MEDIUM = 2; SOXR_QUALITY_HIGH = 3; SOXR_QUALITY_VERYHIGH = 4; } enum SoxFlagBits { SOXR_ROLLOFF_SMALL = 0; // 1 << 0 SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1 SOXR_ROLLOFF_NONE = 2; // 1 << 2 SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3 SOXR_DOUBLE_PRECISION = 4; // 1 << 4 SOXR_VR = 5; // 1 << 5 } // // AudioFrame buffer // message AudioFrameBufferInfo { required uint64 data_ptr = 1; // *const i16 required uint32 num_channels = 2; required uint32 sample_rate = 3; required uint32 samples_per_channel = 4; } message OwnedAudioFrameBuffer { required FfiOwnedHandle handle = 1; required AudioFrameBufferInfo info = 2; } // // AudioStream // enum AudioStreamType { AUDIO_STREAM_NATIVE = 0; AUDIO_STREAM_HTML = 1; } message AudioStreamInfo { required AudioStreamType type = 1; } message OwnedAudioStream { required FfiOwnedHandle handle = 1; required AudioStreamInfo info = 2; } message AudioStreamEvent { required uint64 stream_handle = 1; oneof message { AudioFrameReceived frame_received = 2; AudioStreamEOS eos = 3; } } message AudioFrameReceived { required OwnedAudioFrameBuffer frame = 1; } message AudioStreamEOS {} // // AudioSource // message AudioSourceOptions { required bool echo_cancellation = 1; required bool noise_suppression = 2; required bool auto_gain_control = 3; } enum AudioSourceType { AUDIO_SOURCE_NATIVE = 0; } message AudioSourceInfo { required AudioSourceType type = 2; } message OwnedAudioSource { required FfiOwnedHandle handle = 1; required AudioSourceInfo info = 2; } // // AudioResampler // message AudioResamplerInfo { } message OwnedAudioResampler { required FfiOwnedHandle handle = 1; required AudioResamplerInfo info = 2; } // // AEC // message OwnedApm { required FfiOwnedHandle handle = 1; } // // Sox AudioResampler // message SoxResamplerInfo {} message OwnedSoxResampler { required FfiOwnedHandle handle = 1; required SoxResamplerInfo info = 2; } // Audio Filter Plugin message LoadAudioFilterPluginRequest { required string plugin_path = 1; // path for ffi audio filter plugin repeated string dependencies = 2; // Optional: paths for dependency dylibs required string module_id = 3; // Unique identifier of the plugin } message LoadAudioFilterPluginResponse { optional string error = 1; }