From: Max Value Date: Fri, 10 Jan 2025 13:00:24 +0000 (+0000) Subject: Rehashing audio system. Added transformers X-Git-Url: https://git.ozva.co.uk/?a=commitdiff_plain;h=b53bef37348f5334e8acde7d4040f5a0ebd48812;p=rust_fft Rehashing audio system. Added transformers --- diff --git a/build.rs b/build.rs index 771e13d..dd3bf66 100644 --- a/build.rs +++ b/build.rs @@ -5,7 +5,7 @@ fn main() { cc::Build::new() .file("src/perspective.cpp") .cpp(true) - .include("/usr/share/include/opencv4/") + .include("/usr/share/include/opencv4/opencv2/") .compile("perspective.a"); println!("cargo::rustc-flags=-lopencv_core -lopencv_highgui -lopencv_xfeatures2d -lopencv_calib3d -lopencv_videoio -lopencv_imgcodecs -lopencv_imgproc -lopencv_features2d"); diff --git a/src/camera.rs b/src/camera.rs index 5c84766..7de72f5 100644 --- a/src/camera.rs +++ b/src/camera.rs @@ -69,25 +69,41 @@ impl ImageArray { } pub fn from_buffer (&mut self, buffer: &Vec>) -> () { + let mut r: f32; + let mut theta: f32; + let mut amplitude: f32; + + let mut hue: f32; + let mut angle: f32; + + let mut d:f32; + let mut s:f32; + let mut v:f32; + let mut c:f32; + let mut m:f32; + let mut x:f32; + let mut g:f32; + let mut b:f32; + for i in 0..self.chunks { - let (r, theta): (f32, f32) = buffer[i].to_polar(); + (r, theta) = buffer[i].to_polar(); - let amplitude = 20f32 * r.log10(); - let amplitude = ((amplitude - VOLUME_MIN) / (VOLUME_REL / AMPLITUDE_REL)) + AMPLITUDE_MIN; + amplitude = 20f32 * r.log10(); + amplitude = ((amplitude - VOLUME_MIN) / (VOLUME_REL / AMPLITUDE_REL)) + AMPLITUDE_MIN; - let hue = (180f32 / 255f32) * amplitude; + hue = (180f32 / 255f32) * amplitude; - let angle = (theta.to_degrees() + 180f32) * (ANGLE_REL / 360f32) + ANGLE_MIN; + angle = (theta.to_degrees() + 180f32) * (ANGLE_REL / 360f32) + ANGLE_MIN; - let d = hue * (1f32 / 30f32); - let s = angle / 255f32; - let v = amplitude / 255f32; + d = hue * (1f32 / 30f32); + s = angle / 255f32; + v = amplitude / 255f32; - let c = s * v; - let m = v - c; - let x = c * (1f32 - (d.rem_euclid(2f32) - 1f32).abs()); + c = s * v; + m = v - c; + x = c * (1f32 - (d.rem_euclid(2f32) - 1f32).abs()); - let (r, g, b) = match d.floor() { + (r, g, b) = match d.floor() { 0.0 => (c, x, 0f32), 1.0 => (x, c, 0f32), 2.0 => (0f32, c, x), @@ -103,21 +119,33 @@ impl ImageArray { } pub fn to_buffer (&mut self, buffer: &mut Vec>) -> () { + let mut r: f32; + let mut amplitude: f32; + + let mut hue: f32; + let mut angle: f32; + + let mut s:f32; + let mut v:f32; + let mut c:f32; + let mut g:f32; + let mut b:f32; + for i in 0..self.chunks { - let r = self.data[i*3] as f32; - let g = self.data[i*3+1] as f32; - let b = self.data[i*3+2] as f32; + r = self.data[i*3] as f32; + g = self.data[i*3+1] as f32; + b = self.data[i*3+2] as f32; - let v = r.max(g).max(b); - let c = (v - r.min(g).min(b)) * 255f32; - let s = if v == 0f32 { 0f32 } else { c / v }; + v = r.max(g).max(b); + c = (v - r.min(g).min(b)) * 255f32; + s = if v == 0f32 { 0f32 } else { c / v }; - let amplitude = (v - AMPLITUDE_MIN) * (VOLUME_REL / AMPLITUDE_REL) + VOLUME_MIN; + amplitude = (v - AMPLITUDE_MIN) * (VOLUME_REL / AMPLITUDE_REL) + VOLUME_MIN; - let amplitude = 10f32.powf(amplitude / 20f32); + amplitude = 10f32.powf(amplitude / 20f32); - let angle = (s - ANGLE_MIN) / (ANGLE_REL / 360f32) - 180f32; - let angle = angle.to_radians(); + angle = (s - ANGLE_MIN) / (ANGLE_REL / 360f32) - 180f32; + angle = angle.to_radians(); buffer[i] = Complex::from_polar(amplitude, angle); } diff --git a/src/main.rs b/src/main.rs index d310fbd..2178eff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ use show_image::{ImageView, ImageInfo, create_window, event}; use hound; use cpal::{StreamConfig, BufferSize, SampleRate}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use sound::{SoundObject, Location}; use std::sync::{Arc, Mutex}; use std::sync::mpsc; use std::sync::mpsc::Sender; @@ -12,6 +13,7 @@ use image::ImageReader; pub mod camera; pub mod sound; +pub mod transformer; use crate::camera::ImageArray; @@ -44,41 +46,6 @@ const VOLUME_REL: f32 = VOLUME_MAX - VOLUME_MIN; const DEBUG_MODE: bool = true; -struct SampleBuffer { - buffer: Arc>, - index: usize, - tx: Sender -} - -impl SampleBuffer { - fn new(buffer: Arc>, tx: Sender) -> Self { - Self { - buffer, - index: 0, - tx - } - } - - fn get_data(&mut self, data: &mut [i16]) { - let mut buffer = self.buffer.lock().unwrap(); - let length = data.len()/2; - - for i in 0..length { - data[i*2] = buffer[i + self.index]; - } - - self.index += length; - if self.index > SPECTOGRAM_AREA { - for i in 0..SPECTOGRAM_AREA { - buffer[i] = buffer[i + SPECTOGRAM_AREA]; - } - self.index -= SPECTOGRAM_AREA; - - let _ = self.tx.send(true); - } - } -} - #[show_image::main] fn main () -> Result<(), Box> { // pregenerate the fft transformers @@ -90,11 +57,6 @@ fn main () -> Result<(), Box> { let mut buffer = vec![Complex{re: 0f32, im: 0f32}; SPECTOGRAM_AREA]; let mut scratch = vec![Complex{re: 0f32, im: 0f32}; scratch_size]; - // setup communication between the main thread and the audio thread - let (tx, rx) = mpsc::channel(); - let sample_buffer = Arc::new(Mutex::new([0i16; 2*SPECTOGRAM_AREA])); - let mut samples = SampleBuffer::new(sample_buffer.clone(), tx); - // setup homography space and get ready for the calibration let homography = [0f64; 9]; // homography is a 3x3 matrix of 64-bit floats let mut image_array = ImageArray::new(homography); @@ -119,76 +81,50 @@ fn main () -> Result<(), Box> { // calibrate camera image_array.calibrate(); - // open audio file - let mut reader = hound::WavReader::open("/home/will/Downloads/Adducci - Around the Horn.wav").unwrap(); - let file_rate = reader.spec().sample_rate; - - // setup audio output and build output stream - let host = cpal::default_host(); - let device = host.default_output_device().expect("No output device available"); - - let stream = device.build_output_stream( - &StreamConfig{ - channels: 2, - sample_rate: SampleRate{0: file_rate}, - buffer_size: BufferSize::Fixed(WINDOW_SIZE as u32) - }, - move |data: &mut [i16], _: &cpal::OutputCallbackInfo| { - samples.get_data(data); - }, - move |err| { - eprintln!("an error occurred on the output audio stream: {}", err); - }, - None - ).unwrap(); - stream.play().expect("Stream play failed"); - - // begin looping though the samples until the buffer is full of complex audio with an imag element of 0 - let mut i = 0; - for sample in reader.samples::() { - if i == SPECTOGRAM_AREA { - - // transform and do power scaling - forward_transform.process_with_scratch(&mut buffer, &mut scratch); - for x in buffer.iter_mut() { - *x *= 1f32 / WINDOW_SIZE as f32; - } - image_array.from_buffer(&mut buffer); + // get the input and output devices ready + let config = StreamConfig { + channels: 1, + sample_rate: SampleRate(48_000), + buffer_size: BufferSize::Default + }; + let input = Location::File(String::from("/home/will/Music/Adducci - Around the Horn.wav")); + let output = Location::Physical(config); + let sound_manager = SoundObject::new(input, output); + + loop { + // transform and do power scaling + forward_transform.process_with_scratch(&mut buffer, &mut scratch); + for x in buffer.iter_mut() { + *x *= 1f32 / WINDOW_SIZE as f32; + } + image_array.from_buffer(&mut buffer); - // show the image on screen - let image = ImageView::new(ImageInfo::rgb8(WINDOW_SIZE as u32, CHUNK_SIZE as u32), &image_array.data); - display_window.set_image ("image", image)?; + // show the image on screen + let image = ImageView::new(ImageInfo::rgb8(WINDOW_SIZE as u32, CHUNK_SIZE as u32), &image_array.data); + display_window.set_image ("image", image)?; - // capture and transform camera view to image - image_array.from_camera(); + // capture and transform camera view to image + image_array.from_camera(); - // make and display debug image - if DEBUG_MODE { - let debug_image = ImageView::new(ImageInfo::rgb8(WINDOW_SIZE as u32, CHUNK_SIZE as u32), &image_array.data); - debug_window.set_image("Debug", debug_image)?; - } + // make and display debug image + if DEBUG_MODE { + let debug_image = ImageView::new(ImageInfo::rgb8(WINDOW_SIZE as u32, CHUNK_SIZE as u32), &image_array.data); + debug_window.set_image("Debug", debug_image)?; + } - // convert image to audio - image_array.to_buffer(&mut buffer); - inverse_transform.process_with_scratch(&mut *buffer, &mut scratch); + // convert image to audio + image_array.to_buffer(&mut buffer); + inverse_transform.process_with_scratch(&mut *buffer, &mut scratch); - // when a "true" is receved by rx, get lock for sample buffer and fill the last half with the audio - if rx.recv().unwrap() { - let mut write_buffer = sample_buffer.lock().unwrap(); - for (i, x) in write_buffer[SPECTOGRAM_AREA..].iter_mut().enumerate() { - *x = buffer[i].re as i16; - } + // when a "true" is receved by rx, get lock for sample buffer and fill the last half with the audio + if rx.recv().unwrap() { + let mut write_buffer = sample_buffer.lock().unwrap(); + for (i, x) in write_buffer[SPECTOGRAM_AREA..].iter_mut().enumerate() { + *x = buffer[i].re as i16; } - - i = 0; } - let value = match sample { - Ok(t) => t, - Err(_) => 0i16 - }; - // if buffer is not full convert value and add to buffer - buffer[i] = Complex{re: value as f32, im: 0f32}; - i += 1; + + i = 0; } Ok(()) diff --git a/src/sound.rs b/src/sound.rs index f0f4296..1c715bb 100644 --- a/src/sound.rs +++ b/src/sound.rs @@ -11,11 +11,12 @@ use hound::{ WavWriter, WavSpec, SampleFormat}; use crate::SPECTOGRAM_AREA; -const DUMMY_READ_DELAY: u64 = 1; // seconds +const DUMMY_READ_DELAY: u64 = 1; // Seconds const DUMMY_SAMPLE_SIZE: usize = 10; const DEFAULT_SAMPLE_RATE: u32 = 48_000; +// thread safe object for holding data in a buffer to be taken out by the program struct InputBuffer { buffer: Arc> } @@ -34,7 +35,7 @@ impl InputBuffer { } } - fn give_data(&mut self, data: &mut [i16]) { + fn give_data(&mut self, data: &[i16]) { let mut buffer = self.buffer.lock().unwrap(); let length = data.len()/2; @@ -48,6 +49,7 @@ impl InputBuffer { } } +// thread safe object for holding data in a buffer bound for the sound device struct OutputBuffer { buffer: Arc>, index: usize, @@ -86,7 +88,7 @@ impl OutputBuffer { } fn give_data(&mut self, data: &[Complex]) { - if self.rx.recv().unwrap() { + if self.rx.recv().unwrap() { // locking until the current stream is done is included by default let mut buffer = self.buffer.lock().unwrap(); for (i, x) in buffer[SPECTOGRAM_AREA..].iter_mut().enumerate() { *x = data[i].re as i16; @@ -95,20 +97,28 @@ impl OutputBuffer { } } -enum Location { +pub enum FileData<'wr> { + FilePath(String), + Data( + hound::WavReader>, + hound::WavSamples<'wr, std::io::BufReader, i16> + ) +} + +pub enum Location<'wr> { Physical(StreamConfig), - File(String), + File(FileData<'wr>), None } -pub struct SoundObject { - callback: fn(&mut [i16]), +pub struct SoundObject<'wr> { + input: Location<'wr>, input_buffer: InputBuffer, output_buffer: OutputBuffer } -impl SoundObject { - fn new(input: Location, output: Location) -> Self { +impl SoundObject<'_> { + pub fn new(input: Location, output: Location) -> Self { let input_buffer_space = Arc::new(Mutex::new([0i16; SPECTOGRAM_AREA])); let input_buffer = InputBuffer::new(input_buffer_space.clone()); let thread_input_buffer = InputBuffer::new(input_buffer_space); @@ -117,13 +127,13 @@ impl SoundObject { let output_buffer = OutputBuffer::new(output_buffer_space.clone()); let thread_output_buffer = OutputBuffer::new(output_buffer_space); - let (sample_rate, callback):(u32, Box) = match input { + let sample_rate: u32 = match input { Location::Physical(config) => { let host = cpal::default_host(); let device = host.default_input_device().expect("No input device available"); let stream = device.build_input_stream( &config, - move | data: &mut [i16], _: &InputCallbackInfo | { + move | data: & [i16], _: &InputCallbackInfo | { thread_input_buffer.give_data(data); }, move | err | { @@ -133,43 +143,26 @@ impl SoundObject { ).unwrap(); stream.play().expect("Output stream play failed"); - let callback = Box::new(| data: &mut [i16] | { - input_buffer.get_data(data); - }); - (DEFAULT_SAMPLE_RATE, callback) + config.sample_rate.0 }, - Location::File(string) => { + Location::File(FileData::FilePath(string)) => { let mut reader = hound::WavReader::open(string).unwrap(); let sample_rate = reader.spec().sample_rate; - let mut samples = reader.samples(); - let callback = Box::new(| data: &mut [i16] | { - for sample in data.iter_mut() { - *sample = match samples.next() { - Some(x) => x.unwrap(), - None => { - samples = reader.samples(); - 0 - } - } as i16 - } - }); - (sample_rate, callback) - }, - Location::None => { - let callback = Box::new(| data: &mut [i16] | { - for point in data.iter_mut() { - *point = 0i16; - } - }); - (DEFAULT_SAMPLE_RATE, callback) - } + input = Location::File(FileData::Data(reader, reader.samples())); + + sample_rate + + }, + Location::File(FileData::Data(_, _)) => {DEFAULT_SAMPLE_RATE}, + Location::None => {DEFAULT_SAMPLE_RATE} }; match output { Location::Physical(config) => { let host = cpal::default_host(); let device = host.default_output_device().expect("No output device available"); + // config.sample_rate.0 = sample_rate; // make output rate same as input rate let stream = device.build_output_stream( &config, move |data: &mut [i16], _: &OutputCallbackInfo| { @@ -182,7 +175,7 @@ impl SoundObject { ).unwrap(); stream.play().expect("Output stream play failed"); }, - Location::File(path) => { + Location::File(FileData::FilePath(path)) => { let mut writer = WavWriter::create(path, WavSpec{ channels: 1, sample_rate, @@ -201,6 +194,7 @@ impl SoundObject { } }); }, + Location::File(FileData::Data(_, _)) => {panic!("Unexpected error");} Location::None => { let mut data = [0i16; DUMMY_SAMPLE_SIZE]; let write_delay = DUMMY_SAMPLE_SIZE as f32 / sample_rate as f32; @@ -211,14 +205,46 @@ impl SoundObject { } }); } - } + }; Self { - callback, + input, input_buffer, output_buffer } } + + pub fn get_data(&mut self, data: &mut [i16]) { + match &mut self.input { + Location::Physical(_) => { + self.input_buffer.get_data(data); + }, + Location::File(FileData::Data(reader, samples)) => { + for sample in data.iter_mut() { + *sample = match samples.next() { + Some(x) => x.unwrap(), + None => { + reader = &mut reader.samples(); + 0 + } + } as i16; + } + }, + Location::File(FileData::FilePath(_)) => { + for point in data.iter_mut() { + *point = 0i16; + } + }, + Location::None => { + for point in data.iter_mut() { + *point = 0i16; + } + } + } + } + pub fn give_data(&mut self, data: &[Complex]) { + self.output_buffer.give_data(data); + } } diff --git a/src/transformer.rs b/src/transformer.rs new file mode 100644 index 0000000..f7a43e9 --- /dev/null +++ b/src/transformer.rs @@ -0,0 +1,9 @@ + + +pub struct Transformer {} + +impl Transformer { + fn new() -> Self { + + } +}