diff --git a/Cargo.lock b/Cargo.lock index 94a57d8..cc679a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,6 +113,39 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "audio-codec-algorithms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1254ebf6529f3763c491acfb5ab6e960809e3c75a38584cc664f8c5667fb7107" + +[[package]] +name = "audioadapter" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1292ef9edf681b7426ed089004b021a42896492f58bd0c909ad44e5faec9ac4" + +[[package]] +name = "audioadapter-buffers" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46289a81a3bfa26d0f8b2415f8ca9decde9a66308a150568af5c102381b3c8ce" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5258faecf4edbe35ec483bda9b1f7438e6b391891819269c6b5461aeaa0e8d7d" +dependencies = [ + "audio-codec-algorithms", + "num-traits", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -280,6 +313,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" name = "client" version = "0.1.0" dependencies = [ + "audioadapter-buffers", "clap", "cpal", "crossterm", @@ -289,6 +323,7 @@ dependencies = [ "ratatui", "ringbuf", "ropus", + "rubato", "serde", "serde_json", "shared", @@ -1128,6 +1163,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1145,6 +1189,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1479,6 +1532,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -1651,6 +1713,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1710,6 +1781,22 @@ dependencies = [ "wide", ] +[[package]] +name = "rubato" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cb1ffaf8738df50aab642a7f6465df81c6ba9e2818268053487165298114be" +dependencies = [ + "audioadapter", + "audioadapter-buffers", + "num-complex", + "num-integer", + "num-traits", + "realfft", + "visibility", + "windowfunctions", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -1719,6 +1806,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1936,6 +2037,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strsim" version = "0.11.1" @@ -2218,6 +2325,16 @@ dependencies = [ "winnow", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "typenum" version = "1.20.1" @@ -2283,6 +2400,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "vtparse" version = "0.6.2" @@ -2485,6 +2613,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.62.2" diff --git a/client/Cargo.toml b/client/Cargo.toml index f053389..6766d4e 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -18,3 +18,5 @@ crossterm = "0.29.0" gag = "1.0.0" tokio-util = { version = "0.7.19", features = ["codec"]} futures = "0.3.34" +rubato = "5.0.0" +audioadapter-buffers = "5.2.0" diff --git a/client/src/audio_streams.rs b/client/src/audio_streams.rs index 1d85176..f3b7617 100644 --- a/client/src/audio_streams.rs +++ b/client/src/audio_streams.rs @@ -1,5 +1,7 @@ +use audioadapter_buffers::direct::InterleavedSlice; use cpal::{ - Device, FromSample, Sample, SampleFormat, SizedSample, StreamConfig, SupportedStreamConfig, + Device, FromSample, SAMPLE_RATE_48K, Sample, SampleFormat, SizedSample, StreamConfig, + SupportedStreamConfig, traits::{DeviceTrait, StreamTrait}, }; use ringbuf::{ @@ -9,6 +11,7 @@ use ringbuf::{ wrap::caching::Caching, }; use ropus::Encoder; +use rubato::{Async, FixedAsync, PolynomialDegree, Resampler, audioadapter::Adapter}; use shared::AudioStreamInfo; use std::sync::{Arc, RwLock}; use tokio::{ @@ -318,16 +321,26 @@ pub async fn audio_stream_spawner( input_stream.play().expect("failed to play input stream"); - let samples_needed = ((20.0 / 1000.0) - * input_device_config.sample_rate() as f32 - * input_device_config.channels() as f32) as usize; + let frames_needed = ((20.0 / 1000.0) * input_device_config.sample_rate() as f32) as usize; - let output_samples_needed = ((20.0 / 1000.0) - * input_device_config.sample_rate() as f32 - * force_channels as f32) as usize; + let samples_needed = frames_needed * input_device_config.channels() as usize; let mut frame_buffer = vec![0f32; samples_needed]; - let mut output_frame_buffer = vec![0f32; output_samples_needed]; + + let mut resampler_input_buffer = vec![0.0f32; frames_needed * force_channels as usize]; + + let mut resampler = Async::::new_poly( + 48000.0 / 44100.0, + 1.1, + PolynomialDegree::Cubic, + frames_needed, + force_channels as usize, + FixedAsync::Output, + ) + .unwrap(); + + let mut resampler_output_buffer = + vec![0.0f32; resampler.output_frames_max() * force_channels as usize]; let encoder_channels: ropus::Channels = match force_channels { 1 => ropus::Channels::Mono, @@ -338,13 +351,10 @@ pub async fn audio_stream_spawner( } }; - let mut opus_encoder = Encoder::builder( - input_device_config.sample_rate(), - encoder_channels, - ropus::Application::Voip, - ) - .build() - .unwrap(); + let mut opus_encoder = + Encoder::builder(SAMPLE_RATE_48K, encoder_channels, ropus::Application::Voip) + .build() + .unwrap(); let mut encoded_frame_buffer = [0u8; 2048]; @@ -352,26 +362,40 @@ pub async fn audio_stream_spawner( if consumer.occupied_len() >= samples_needed { consumer.pop_slice(&mut frame_buffer); - if force_channels == input_device_config.channels() { - output_frame_buffer.copy_from_slice(frame_buffer.as_slice()); - } else if force_channels < input_device_config.channels() { - let div = (input_device_config.channels() / force_channels) as usize; + let frame_buffer_adapter = InterleavedSlice::new( + &frame_buffer, + input_device_config.channels() as usize, + frames_needed, + ) + .unwrap(); - output_frame_buffer.fill(0.0); + frame_buffer_adapter.copy_from_channel_to_slice(0, 0, &mut resampler_input_buffer); // im only using first channel for now - for (i, input_sample) in frame_buffer.iter().enumerate() { - output_frame_buffer[i / div] += *input_sample / div as f32; - } - } else { - let div = (force_channels / input_device_config.channels()) as usize; + let resampler_input = InterleavedSlice::new( + &resampler_input_buffer, + force_channels as usize, + frames_needed, + ) + .unwrap(); - for (i, output_sample) in output_frame_buffer.iter_mut().enumerate() { - *output_sample = frame_buffer[i / div]; - } - } + let mut resampler_output = InterleavedSlice::new_mut( + &mut resampler_output_buffer, + force_channels as usize, + resampler.output_frames_max(), + ) + .unwrap(); + + let (_frames_read, frames_written) = resampler + .process_into_buffer(&resampler_input, &mut resampler_output, None) + .unwrap(); + + let valid_samples = frames_written * force_channels as usize; let encoded_len = opus_encoder - .encode_float(&output_frame_buffer.as_slice(), &mut encoded_frame_buffer) + .encode_float( + &resampler_output_buffer[..valid_samples], + &mut encoded_frame_buffer, + ) .expect("failed to encode frame"); let mut payload = Vec::::with_capacity(4 + 1 + encoded_len * 4); diff --git a/client/src/tui.rs b/client/src/tui.rs index f955928..d0bae39 100644 --- a/client/src/tui.rs +++ b/client/src/tui.rs @@ -123,46 +123,14 @@ impl DeviceSelectScreen { if let Some(default_input_device) = host.default_input_device() { if let Ok(default_config) = default_input_device.default_input_config() { - if default_config.sample_rate() == SAMPLE_RATE_48K - && (default_config.channels() == 1) - { - input_devices.push((default_input_device.clone(), default_config)); - } - } else if let Ok(supported_configs) = default_input_device.supported_input_configs() { - for supported_config in supported_configs { - if let Some(input_config) = - supported_config.try_with_sample_rate(SAMPLE_RATE_48K) - { - if input_config.channels() == 1 { - input_devices.push((default_input_device, input_config)); - break; - } - } - } + input_devices.push((default_input_device.clone(), default_config)); } } if let Ok(possible_input_devices) = host.input_devices() { for possible_input_device in possible_input_devices { if let Ok(default_config) = possible_input_device.default_input_config() { - if default_config.sample_rate() == SAMPLE_RATE_48K - && (default_config.channels() == 1) - { - input_devices.push((possible_input_device.clone(), default_config)); - continue; - } - } - if let Ok(supported_configs) = possible_input_device.supported_input_configs() { - for supported_config in supported_configs { - if let Some(input_config) = - supported_config.try_with_sample_rate(SAMPLE_RATE_48K) - { - if input_config.channels() == 1 { - input_devices.push((possible_input_device, input_config)); - break; - } - } - } + input_devices.push((possible_input_device.clone(), default_config)); } } } @@ -171,47 +139,14 @@ impl DeviceSelectScreen { if let Some(default_output_device) = host.default_output_device() { if let Ok(default_config) = default_output_device.default_output_config() { - if default_config.sample_rate() == SAMPLE_RATE_48K - && (default_config.channels() == 1 || default_config.channels() == 2) - { - output_devices.push((default_output_device.clone(), default_config.into())); - } - } else if let Ok(supported_configs) = default_output_device.supported_output_configs() { - for supported_config in supported_configs { - if let Some(output_config) = - supported_config.try_with_sample_rate(SAMPLE_RATE_48K) - { - if output_config.channels() == 1 || output_config.channels() == 2 { - output_devices.push((default_output_device, output_config.into())); - break; - } - } - } + output_devices.push((default_output_device.clone(), default_config.into())); } } if let Ok(possible_output_devices) = host.output_devices() { for possible_output_device in possible_output_devices { if let Ok(default_config) = possible_output_device.default_output_config() { - if default_config.sample_rate() == SAMPLE_RATE_48K - && (default_config.channels() == 1 || default_config.channels() == 2) - { - output_devices - .push((possible_output_device.clone(), default_config.into())); - continue; - } - } - if let Ok(supported_configs) = possible_output_device.supported_output_configs() { - for supported_config in supported_configs { - if let Some(output_config) = - supported_config.try_with_sample_rate(SAMPLE_RATE_48K) - { - if output_config.channels() == 1 || output_config.channels() == 2 { - output_devices.push((possible_output_device, output_config.into())); - break; - } - } - } + output_devices.push((possible_output_device.clone(), default_config.into())); } } } @@ -261,11 +196,18 @@ impl DeviceSelectScreen { let widths = [Constraint::Percentage(100), Constraint::Min(8)]; let mut input_device_rows: Vec = Vec::new(); - for (device, config) in self.input_devices.iter() { - input_device_rows.push(Row::new(vec![ - device.description().unwrap().name().to_string(), - config.channels().to_string(), - ])); + for (i, (device, config)) in self.input_devices.iter().enumerate() { + if i == 0 { + input_device_rows.push(Row::new(vec![ + format!("(default) {}", device.description().unwrap().name()), + config.channels().to_string(), + ])); + } else { + input_device_rows.push(Row::new(vec![ + device.description().unwrap().name().to_string(), + config.channels().to_string(), + ])); + } } let input_device_table = Table::new(input_device_rows, widths) .header( @@ -296,11 +238,21 @@ impl DeviceSelectScreen { frame.render_stateful_widget(input_device_table, left_block, &mut self.input_device_state); let mut output_device_rows: Vec = Vec::new(); - for (device, config) in self.output_devices.iter() { - output_device_rows.push(Row::new(vec![ - device.description().unwrap().name().to_string(), - config.channels().to_string(), - ])); + for (i, (device, config)) in self.output_devices.iter().enumerate() { + if i == 0 { + output_device_rows.push(Row::new(vec![ + format!( + "(default) {}", + device.description().unwrap().name().to_string() + ), + config.channels().to_string(), + ])); + } else { + output_device_rows.push(Row::new(vec![ + device.description().unwrap().name().to_string(), + config.channels().to_string(), + ])); + } } let output_device_table = Table::new(output_device_rows, widths) .header( diff --git a/server/src/main.rs b/server/src/main.rs index c1b1f2c..223e93a 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -26,8 +26,6 @@ struct Client { #[allow(dead_code)] #[derive(Clone)] enum ServerEvent { - NewAudioStream(AudioStreamInfo), - SendAllStreams, SendAllOtherStreams(u32), UserConnected(u32), UserDisconnected(u32), @@ -71,7 +69,7 @@ async fn main() { let mut buf = vec![0u8; 4096]; loop { - let (len, src_adddress) = udp_socket.recv_from(&mut buf).await.expect("receive error"); + let (len, src_address) = udp_socket.recv_from(&mut buf).await.expect("receive error"); if len < 4 { continue; @@ -82,13 +80,13 @@ async fn main() { let mut clients = udp_state.write().await; if let Some(client) = clients.get_mut(&client_id) { - client.udp_addr = Some(src_adddress); + client.udp_addr = Some(src_address); } for (&other_client_id, other_client) in clients.iter() { if other_client_id != client_id { - if let Some(destication_udp) = other_client.udp_addr { - let _ = udp_socket.send_to(&buf[..len], destication_udp).await; + if let Some(destination_udp) = other_client.udp_addr { + let _ = udp_socket.send_to(&buf[..len], destination_udp).await; } } } @@ -230,21 +228,6 @@ async fn main() { let _ = framed_tcp_stream.send(Bytes::from(response_json)).await; } } - ServerEvent::SendAllStreams => { - let clients = tcp_state.read().await; - - let response_json = serde_json::to_string(&TcpResponse { - status: ResponseStatus::Ok, - data: shared::TcpResponseData::AudioStreamsFullUpdate { - audio_stream_info: scrape_all_client_streams_info(&clients), - }, - }).expect("failed to serialize ServerEvent::SendAllStreams"); - - let _ = framed_tcp_stream.send(Bytes::from(response_json)).await; - } - _ => { - println!("unhandled event"); - } } } Err(e) => eprintln!("error: {}", e),