removing device selection filters; possibly converting to 48khz input

2026-10-19 23:29:00 +02:00
parent 0988ada8ac
commit 687ef4ff2c
5 changed files with 228 additions and 130 deletions
@@ -113,6 +113,39 @@ dependencies = [
 "bytemuck",
 ]

+[[package]]
+name = "audio-codec-algorithms"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1254ebf6529f3763c491acfb5ab6e960809e3c75a38584cc664f8c5667fb7107"
+
+[[package]]
+name = "audioadapter"
+version = "5.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1292ef9edf681b7426ed089004b021a42896492f58bd0c909ad44e5faec9ac4"
+
+[[package]]
+name = "audioadapter-buffers"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46289a81a3bfa26d0f8b2415f8ca9decde9a66308a150568af5c102381b3c8ce"
+dependencies = [
+ "audioadapter",
+ "audioadapter-sample",
+ "num-traits",
+]
+
+[[package]]
+name = "audioadapter-sample"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5258faecf4edbe35ec483bda9b1f7438e6b391891819269c6b5461aeaa0e8d7d"
+dependencies = [
+ "audio-codec-algorithms",
+ "num-traits",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.5.1"
@@ -280,6 +313,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
 name = "client"
 version = "0.1.0"
 dependencies = [
+ "audioadapter-buffers",
 "clap",
 "cpal",
 "crossterm",
@@ -289,6 +323,7 @@ dependencies = [
 "ratatui",
 "ringbuf",
 "ropus",
+ "rubato",
 "serde",
 "serde_json",
 "shared",
@@ -1128,6 +1163,15 @@ dependencies = [
 "minimal-lexical",
 ]

+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.2.2"
@@ -1145,6 +1189,15 @@ dependencies = [
 "syn 2.0.119",
 ]

+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -1479,6 +1532,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"

+[[package]]
+name = "primal-check"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
+dependencies = [
+ "num-integer",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "3.5.0"
@@ -1651,6 +1713,15 @@ dependencies = [
 "unicode-width",
 ]

+[[package]]
+name = "realfft"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677"
+dependencies = [
+ "rustfft",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"
@@ -1710,6 +1781,22 @@ dependencies = [
 "wide",
 ]

+[[package]]
+name = "rubato"
+version = "5.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7cb1ffaf8738df50aab642a7f6465df81c6ba9e2818268053487165298114be"
+dependencies = [
+ "audioadapter",
+ "audioadapter-buffers",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "realfft",
+ "visibility",
+ "windowfunctions",
+]
+
 [[package]]
 name = "rustc_version"
 version = "0.4.1"
@@ -1719,6 +1806,20 @@ dependencies = [
 "semver",
 ]

+[[package]]
+name = "rustfft"
+version = "6.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89"
+dependencies = [
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "primal-check",
+ "strength_reduce",
+ "transpose",
+]
+
 [[package]]
 name = "rustix"
 version = "1.1.4"
@@ -1936,6 +2037,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"

+[[package]]
+name = "strength_reduce"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -2218,6 +2325,16 @@ dependencies = [
 "winnow",
 ]

+[[package]]
+name = "transpose"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
+dependencies = [
+ "num-integer",
+ "strength_reduce",
+]
+
 [[package]]
 name = "typenum"
 version = "1.20.1"
@@ -2283,6 +2400,17 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"

+[[package]]
+name = "visibility"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "vtparse"
 version = "0.6.2"
@@ -2485,6 +2613,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

+[[package]]
+name = "windowfunctions"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "windows"
 version = "0.62.2"
@@ -18,3 +18,5 @@ crossterm = "0.29.0"
 gag = "1.0.0"
 tokio-util = { version = "0.7.19", features = ["codec"]}
 futures = "0.3.34"
+rubato = "5.0.0"
+audioadapter-buffers = "5.2.0"
@@ -1,5 +1,7 @@
+use audioadapter_buffers::direct::InterleavedSlice;
 use cpal::{
-    Device, FromSample, Sample, SampleFormat, SizedSample, StreamConfig, SupportedStreamConfig,
+    Device, FromSample, SAMPLE_RATE_48K, Sample, SampleFormat, SizedSample, StreamConfig,
+    SupportedStreamConfig,
    traits::{DeviceTrait, StreamTrait},
 };
 use ringbuf::{
@@ -9,6 +11,7 @@ use ringbuf::{
    wrap::caching::Caching,
 };
 use ropus::Encoder;
+use rubato::{Async, FixedAsync, PolynomialDegree, Resampler, audioadapter::Adapter};
 use shared::AudioStreamInfo;
 use std::sync::{Arc, RwLock};
 use tokio::{
@@ -318,16 +321,26 @@ pub async fn audio_stream_spawner(

    input_stream.play().expect("failed to play input stream");

-    let samples_needed = ((20.0 / 1000.0)
-        * input_device_config.sample_rate() as f32
-        * input_device_config.channels() as f32) as usize;
+    let frames_needed = ((20.0 / 1000.0) * input_device_config.sample_rate() as f32) as usize;

-    let output_samples_needed = ((20.0 / 1000.0)
-        * input_device_config.sample_rate() as f32
-        * force_channels as f32) as usize;
+    let samples_needed = frames_needed * input_device_config.channels() as usize;

    let mut frame_buffer = vec![0f32; samples_needed];
-    let mut output_frame_buffer = vec![0f32; output_samples_needed];
+
+    let mut resampler_input_buffer = vec![0.0f32; frames_needed * force_channels as usize];
+
+    let mut resampler = Async::<f32>::new_poly(
+        48000.0 / 44100.0,
+        1.1,
+        PolynomialDegree::Cubic,
+        frames_needed,
+        force_channels as usize,
+        FixedAsync::Output,
+    )
+    .unwrap();
+
+    let mut resampler_output_buffer =
+        vec![0.0f32; resampler.output_frames_max() * force_channels as usize];

    let encoder_channels: ropus::Channels = match force_channels {
        1 => ropus::Channels::Mono,
@@ -338,11 +351,8 @@ pub async fn audio_stream_spawner(
        }
    };

-    let mut opus_encoder = Encoder::builder(
-        input_device_config.sample_rate(),
-        encoder_channels,
-        ropus::Application::Voip,
-    )
+    let mut opus_encoder =
+        Encoder::builder(SAMPLE_RATE_48K, encoder_channels, ropus::Application::Voip)
            .build()
            .unwrap();

@@ -352,26 +362,40 @@ pub async fn audio_stream_spawner(
        if consumer.occupied_len() >= samples_needed {
            consumer.pop_slice(&mut frame_buffer);

-            if force_channels == input_device_config.channels() {
-                output_frame_buffer.copy_from_slice(frame_buffer.as_slice());
-            } else if force_channels < input_device_config.channels() {
-                let div = (input_device_config.channels() / force_channels) as usize;
+            let frame_buffer_adapter = InterleavedSlice::new(
+                &frame_buffer,
+                input_device_config.channels() as usize,
+                frames_needed,
+            )
+            .unwrap();

-                output_frame_buffer.fill(0.0);
+            frame_buffer_adapter.copy_from_channel_to_slice(0, 0, &mut resampler_input_buffer); // im only using first channel for now

-                for (i, input_sample) in frame_buffer.iter().enumerate() {
-                    output_frame_buffer[i / div] += *input_sample / div as f32;
-                }
-            } else {
-                let div = (force_channels / input_device_config.channels()) as usize;
+            let resampler_input = InterleavedSlice::new(
+                &resampler_input_buffer,
+                force_channels as usize,
+                frames_needed,
+            )
+            .unwrap();

-                for (i, output_sample) in output_frame_buffer.iter_mut().enumerate() {
-                    *output_sample = frame_buffer[i / div];
-                }
-            }
+            let mut resampler_output = InterleavedSlice::new_mut(
+                &mut resampler_output_buffer,
+                force_channels as usize,
+                resampler.output_frames_max(),
+            )
+            .unwrap();
+
+            let (_frames_read, frames_written) = resampler
+                .process_into_buffer(&resampler_input, &mut resampler_output, None)
+                .unwrap();
+
+            let valid_samples = frames_written * force_channels as usize;

            let encoded_len = opus_encoder
-                .encode_float(&output_frame_buffer.as_slice(), &mut encoded_frame_buffer)
+                .encode_float(
+                    &resampler_output_buffer[..valid_samples],
+                    &mut encoded_frame_buffer,
+                )
                .expect("failed to encode frame");

            let mut payload = Vec::<u8>::with_capacity(4 + 1 + encoded_len * 4);
@@ -123,46 +123,14 @@ impl DeviceSelectScreen {

        if let Some(default_input_device) = host.default_input_device() {
            if let Ok(default_config) = default_input_device.default_input_config() {
-                if default_config.sample_rate() == SAMPLE_RATE_48K
-                    && (default_config.channels() == 1)
-                {
                input_devices.push((default_input_device.clone(), default_config));
            }
-            } else if let Ok(supported_configs) = default_input_device.supported_input_configs() {
-                for supported_config in supported_configs {
-                    if let Some(input_config) =
-                        supported_config.try_with_sample_rate(SAMPLE_RATE_48K)
-                    {
-                        if input_config.channels() == 1 {
-                            input_devices.push((default_input_device, input_config));
-                            break;
-                        }
-                    }
-                }
-            }
        }

        if let Ok(possible_input_devices) = host.input_devices() {
            for possible_input_device in possible_input_devices {
                if let Ok(default_config) = possible_input_device.default_input_config() {
-                    if default_config.sample_rate() == SAMPLE_RATE_48K
-                        && (default_config.channels() == 1)
-                    {
                    input_devices.push((possible_input_device.clone(), default_config));
-                        continue;
-                    }
-                }
-                if let Ok(supported_configs) = possible_input_device.supported_input_configs() {
-                    for supported_config in supported_configs {
-                        if let Some(input_config) =
-                            supported_config.try_with_sample_rate(SAMPLE_RATE_48K)
-                        {
-                            if input_config.channels() == 1 {
-                                input_devices.push((possible_input_device, input_config));
-                                break;
-                            }
-                        }
-                    }
                }
            }
        }
@@ -171,47 +139,14 @@ impl DeviceSelectScreen {

        if let Some(default_output_device) = host.default_output_device() {
            if let Ok(default_config) = default_output_device.default_output_config() {
-                if default_config.sample_rate() == SAMPLE_RATE_48K
-                    && (default_config.channels() == 1 || default_config.channels() == 2)
-                {
                output_devices.push((default_output_device.clone(), default_config.into()));
            }
-            } else if let Ok(supported_configs) = default_output_device.supported_output_configs() {
-                for supported_config in supported_configs {
-                    if let Some(output_config) =
-                        supported_config.try_with_sample_rate(SAMPLE_RATE_48K)
-                    {
-                        if output_config.channels() == 1 || output_config.channels() == 2 {
-                            output_devices.push((default_output_device, output_config.into()));
-                            break;
-                        }
-                    }
-                }
-            }
        }

        if let Ok(possible_output_devices) = host.output_devices() {
            for possible_output_device in possible_output_devices {
                if let Ok(default_config) = possible_output_device.default_output_config() {
-                    if default_config.sample_rate() == SAMPLE_RATE_48K
-                        && (default_config.channels() == 1 || default_config.channels() == 2)
-                    {
-                        output_devices
-                            .push((possible_output_device.clone(), default_config.into()));
-                        continue;
-                    }
-                }
-                if let Ok(supported_configs) = possible_output_device.supported_output_configs() {
-                    for supported_config in supported_configs {
-                        if let Some(output_config) =
-                            supported_config.try_with_sample_rate(SAMPLE_RATE_48K)
-                        {
-                            if output_config.channels() == 1 || output_config.channels() == 2 {
-                                output_devices.push((possible_output_device, output_config.into()));
-                                break;
-                            }
-                        }
-                    }
+                    output_devices.push((possible_output_device.clone(), default_config.into()));
                }
            }
        }
@@ -261,12 +196,19 @@ impl DeviceSelectScreen {
        let widths = [Constraint::Percentage(100), Constraint::Min(8)];

        let mut input_device_rows: Vec<Row> = Vec::new();
-        for (device, config) in self.input_devices.iter() {
+        for (i, (device, config)) in self.input_devices.iter().enumerate() {
+            if i == 0 {
+                input_device_rows.push(Row::new(vec![
+                    format!("(default) {}", device.description().unwrap().name()),
+                    config.channels().to_string(),
+                ]));
+            } else {
                input_device_rows.push(Row::new(vec![
                    device.description().unwrap().name().to_string(),
                    config.channels().to_string(),
                ]));
            }
+        }
        let input_device_table = Table::new(input_device_rows, widths)
            .header(
                Row::new(vec!["device name", "channels"])
@@ -296,12 +238,22 @@ impl DeviceSelectScreen {
        frame.render_stateful_widget(input_device_table, left_block, &mut self.input_device_state);

        let mut output_device_rows: Vec<Row> = Vec::new();
-        for (device, config) in self.output_devices.iter() {
+        for (i, (device, config)) in self.output_devices.iter().enumerate() {
+            if i == 0 {
+                output_device_rows.push(Row::new(vec![
+                    format!(
+                        "(default) {}",
+                        device.description().unwrap().name().to_string()
+                    ),
+                    config.channels().to_string(),
+                ]));
+            } else {
                output_device_rows.push(Row::new(vec![
                    device.description().unwrap().name().to_string(),
                    config.channels().to_string(),
                ]));
            }
+        }
        let output_device_table = Table::new(output_device_rows, widths)
            .header(
                Row::new(vec!["device name", "channels"])
@@ -26,8 +26,6 @@ struct Client {
 #[allow(dead_code)]
 #[derive(Clone)]
 enum ServerEvent {
-    NewAudioStream(AudioStreamInfo),
-    SendAllStreams,
    SendAllOtherStreams(u32),
    UserConnected(u32),
    UserDisconnected(u32),
@@ -71,7 +69,7 @@ async fn main() {
        let mut buf = vec![0u8; 4096];

        loop {
-            let (len, src_adddress) = udp_socket.recv_from(&mut buf).await.expect("receive error");
+            let (len, src_address) = udp_socket.recv_from(&mut buf).await.expect("receive error");

            if len < 4 {
                continue;
@@ -82,13 +80,13 @@ async fn main() {

            let mut clients = udp_state.write().await;
            if let Some(client) = clients.get_mut(&client_id) {
-                client.udp_addr = Some(src_adddress);
+                client.udp_addr = Some(src_address);
            }

            for (&other_client_id, other_client) in clients.iter() {
                if other_client_id != client_id {
-                    if let Some(destication_udp) = other_client.udp_addr {
-                        let _ = udp_socket.send_to(&buf[..len], destication_udp).await;
+                    if let Some(destination_udp) = other_client.udp_addr {
+                        let _ = udp_socket.send_to(&buf[..len], destination_udp).await;
                    }
                }
            }
@@ -230,21 +228,6 @@ async fn main() {
                                            let _ = framed_tcp_stream.send(Bytes::from(response_json)).await;
                                        }
                                    }
-                                    ServerEvent::SendAllStreams => {
-                                        let clients = tcp_state.read().await;
-
-                                        let response_json = serde_json::to_string(&TcpResponse {
-                                            status: ResponseStatus::Ok,
-                                            data: shared::TcpResponseData::AudioStreamsFullUpdate {
-                                                audio_stream_info: scrape_all_client_streams_info(&clients),
-                                            },
-                                        }).expect("failed to serialize ServerEvent::SendAllStreams");
-
-                                        let _ = framed_tcp_stream.send(Bytes::from(response_json)).await;
-                                    }
-                                    _ => {
-                                        println!("unhandled event");
-                                    }
                                }
                            }
                            Err(e) => eprintln!("error: {}", e),