From 28e43d29cba599b4c34448b89160638f0e57bd32 Mon Sep 17 00:00:00 2001 From: will Date: Sun, 3 Nov 2024 22:23:15 +0000 Subject: [PATCH] Refactoring - Created camera module to deal with camera stuff - Created sound modue to abstract inputs and outputs of different types to indivisual sound objects - changed calibration image path to include src --- src/camera.rs | 125 ++++++++++++++++++++++++ src/main.rs | 115 ++--------------------- src/perspective.cpp | 2 +- src/sound.rs | 224 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 356 insertions(+), 110 deletions(-) create mode 100644 src/camera.rs create mode 100644 src/sound.rs diff --git a/src/camera.rs b/src/camera.rs new file mode 100644 index 0000000..5c84766 --- /dev/null +++ b/src/camera.rs @@ -0,0 +1,125 @@ +use rscam::{Camera, Config}; +use rustfft::num_complex::Complex; + +use crate::SPECTOGRAM_AREA; + +use crate::IMAGE_WIDTH; +use crate::IMAGE_HEIGHT; +use crate::IMAGE_AREA; +use crate::FPS; + +use crate::AMPLITUDE_MIN; +use crate::AMPLITUDE_REL; + +use crate::ANGLE_MIN; +use crate::ANGLE_REL; + +use crate::VOLUME_MIN; +use crate::VOLUME_REL; + +extern "C" { + fn GetHomography(camera_ptr: usize, homography_ptr: usize); + fn ApplyHomography(camera_ptr: usize, buffer_ptr: usize, homography_ptr: usize); +} + +pub struct ImageArray { + pub data: Vec, + homography: [f64; 9], + camera_buffer: Vec, + camera: Camera, + chunks: usize +} + +impl ImageArray { + pub fn new (homography: [f64; 9]) -> Self { + let mut array = Self { + data: vec![0u8; SPECTOGRAM_AREA * 3], + homography, + camera_buffer: vec![0u8; IMAGE_AREA], + + camera: Camera::new("/dev/video0").unwrap(), + + chunks: SPECTOGRAM_AREA + }; + array.camera.start(&Config { + interval: (1, FPS as u32), + resolution: (IMAGE_WIDTH as u32, IMAGE_HEIGHT as u32), + format: b"RGB3", + ..Default::default() + }).unwrap(); + array + } + + pub fn from_camera (&mut self) { + self.camera_buffer = self.camera.capture().unwrap()[..].try_into().expect("Image is wrong size"); + + unsafe{ApplyHomography (self.camera_buffer.as_ptr() as usize, self.data.as_ptr() as usize, self.homography.as_ptr() as usize);} + } + + pub fn calibrate (&mut self) { + // rscam gives and empty image if its not prompted a couple times + for _i in 0..10 { + self.camera_buffer = self.camera.capture().unwrap()[..].try_into().expect("Image is wrong size"); + } + + // enter unsafe and get the homography array + unsafe { + GetHomography(self.camera_buffer.as_ptr() as usize, self.homography.as_ptr() as usize); + } + } + + pub fn from_buffer (&mut self, buffer: &Vec>) -> () { + for i in 0..self.chunks { + let (r, theta): (f32, f32) = buffer[i].to_polar(); + + let amplitude = 20f32 * r.log10(); + let amplitude = ((amplitude - VOLUME_MIN) / (VOLUME_REL / AMPLITUDE_REL)) + AMPLITUDE_MIN; + + let hue = (180f32 / 255f32) * amplitude; + + let angle = (theta.to_degrees() + 180f32) * (ANGLE_REL / 360f32) + ANGLE_MIN; + + let d = hue * (1f32 / 30f32); + let s = angle / 255f32; + let v = amplitude / 255f32; + + let c = s * v; + let m = v - c; + let x = c * (1f32 - (d.rem_euclid(2f32) - 1f32).abs()); + + let (r, g, b) = match d.floor() { + 0.0 => (c, x, 0f32), + 1.0 => (x, c, 0f32), + 2.0 => (0f32, c, x), + 3.0 => (0f32, x, c), + 4.0 => (x, 0f32, c), + _ => (c, 0f32, x) + }; + + self.data[i*3] = ((r + m) * 255f32) as u8; + self.data[i*3+1] = ((g + m) * 255f32) as u8; + self.data[i*3+2] = ((b + m) * 255f32) as u8; + } + } + + pub fn to_buffer (&mut self, buffer: &mut Vec>) -> () { + for i in 0..self.chunks { + let r = self.data[i*3] as f32; + let g = self.data[i*3+1] as f32; + let b = self.data[i*3+2] as f32; + + let v = r.max(g).max(b); + let c = (v - r.min(g).min(b)) * 255f32; + let s = if v == 0f32 { 0f32 } else { c / v }; + + let amplitude = (v - AMPLITUDE_MIN) * (VOLUME_REL / AMPLITUDE_REL) + VOLUME_MIN; + + let amplitude = 10f32.powf(amplitude / 20f32); + + let angle = (s - ANGLE_MIN) / (ANGLE_REL / 360f32) - 180f32; + let angle = angle.to_radians(); + + buffer[i] = Complex::from_polar(amplitude, angle); + } + } +} diff --git a/src/main.rs b/src/main.rs index e0b5d2f..d310fbd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,9 +8,13 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use std::sync::{Arc, Mutex}; use std::sync::mpsc; use std::sync::mpsc::Sender; -use rscam::{Camera, Config}; use image::ImageReader; +pub mod camera; +pub mod sound; + +use crate::camera::ImageArray; + const WINDOW_SIZE: usize = 128; const CHUNK_SIZE: usize = 72; const SPECTOGRAM_AREA: usize = WINDOW_SIZE * CHUNK_SIZE; @@ -40,113 +44,6 @@ const VOLUME_REL: f32 = VOLUME_MAX - VOLUME_MIN; const DEBUG_MODE: bool = true; -extern "C" { - fn GetHomography(camera_ptr: usize, homography_ptr: usize); - fn ApplyHomography(camera_ptr: usize, buffer_ptr: usize, homography_ptr: usize); -} - -struct ImageArray { - data: Vec, - homography: [f64; 9], - camera_buffer: Vec, - camera: Camera, - chunks: usize -} - -impl ImageArray { - fn new (homography: [f64; 9]) -> Self { - let mut array = Self { - data: vec![0u8; SPECTOGRAM_AREA * 3], - homography, - camera_buffer: vec![0u8; IMAGE_AREA], - - camera: Camera::new("/dev/video0").unwrap(), - - chunks: SPECTOGRAM_AREA - }; - array.camera.start(&Config { - interval: (1, FPS as u32), - resolution: (IMAGE_WIDTH as u32, IMAGE_HEIGHT as u32), - format: b"RGB3", - ..Default::default() - }).unwrap(); - array - } - - fn from_camera (&mut self) { - self.camera_buffer = self.camera.capture().unwrap()[..].try_into().expect("Image is wrong size"); - - unsafe{ApplyHomography (self.camera_buffer.as_ptr() as usize, self.data.as_ptr() as usize, self.homography.as_ptr() as usize);} - } - - fn calibrate (&mut self) { - // rscam gives and empty image if its not prompted a couple times - for _i in 0..10 { - self.camera_buffer = self.camera.capture().unwrap()[..].try_into().expect("Image is wrong size"); - } - - // enter unsafe and get the homography array - unsafe { - GetHomography(self.camera_buffer.as_ptr() as usize, self.homography.as_ptr() as usize); - } - } - - fn from_buffer (&mut self, buffer: &Vec>) -> () { - for i in 0..self.chunks { - let (r, theta): (f32, f32) = buffer[i].to_polar(); - - let amplitude = 20f32 * r.log10(); - let amplitude = ((amplitude - VOLUME_MIN) / (VOLUME_REL / AMPLITUDE_REL)) + AMPLITUDE_MIN; - - let hue = (180f32 / 255f32) * amplitude; - - let angle = (theta.to_degrees() + 180f32) * (ANGLE_REL / 360f32) + ANGLE_MIN; - - let d = hue * (1f32 / 30f32); - let s = angle / 255f32; - let v = amplitude / 255f32; - - let c = s * v; - let m = v - c; - let x = c * (1f32 - (d.rem_euclid(2f32) - 1f32).abs()); - - let (r, g, b) = match d.floor() { - 0.0 => (c, x, 0f32), - 1.0 => (x, c, 0f32), - 2.0 => (0f32, c, x), - 3.0 => (0f32, x, c), - 4.0 => (x, 0f32, c), - _ => (c, 0f32, x) - }; - - self.data[i*3] = ((r + m) * 255f32) as u8; - self.data[i*3+1] = ((g + m) * 255f32) as u8; - self.data[i*3+2] = ((b + m) * 255f32) as u8; - } - } - - fn to_buffer (&mut self, buffer: &mut Vec>) -> () { - for i in 0..self.chunks { - let r = self.data[i*3] as f32; - let g = self.data[i*3+1] as f32; - let b = self.data[i*3+2] as f32; - - let v = r.max(g).max(b); - let c = (v - r.min(g).min(b)) * 255f32; - let s = if v == 0f32 { 0f32 } else { c / v }; - - let amplitude = (v - AMPLITUDE_MIN) * (VOLUME_REL / AMPLITUDE_REL) + VOLUME_MIN; - - let amplitude = 10f32.powf(amplitude / 20f32); - - let angle = (s - ANGLE_MIN) / (ANGLE_REL / 360f32) - 180f32; - let angle = angle.to_radians(); - - buffer[i] = Complex::from_polar(amplitude, angle); - } - } -} - struct SampleBuffer { buffer: Arc>, index: usize, @@ -207,7 +104,7 @@ fn main () -> Result<(), Box> { // create window for displaying images and display calibration image let display_window = create_window("Display", Default::default())?; - let calibration_image = ImageReader::open("calibration.jpg")?.decode()?; + let calibration_image = ImageReader::open("src/calibration.jpg")?.decode()?; display_window.set_image("Display", calibration_image)?; // wait for the user to press a key before continuing diff --git a/src/perspective.cpp b/src/perspective.cpp index 493a61d..b88b394 100644 --- a/src/perspective.cpp +++ b/src/perspective.cpp @@ -24,7 +24,7 @@ extern "C" { try { - Mat img1 = imread( samples::findFile("calibration.jpg")/*, IMREAD_GRAYSCALE */); + Mat img1 = imread( samples::findFile("src/calibration.jpg")/*, IMREAD_GRAYSCALE */); Mat img2(IMAGE_HEIGHT, IMAGE_WIDTH, CV_8UC3, camera_ptr); // detect keypoints and compute descriptors diff --git a/src/sound.rs b/src/sound.rs new file mode 100644 index 0000000..f0f4296 --- /dev/null +++ b/src/sound.rs @@ -0,0 +1,224 @@ +use cpal::{StreamConfig, InputCallbackInfo, OutputCallbackInfo}; +use cpal::traits::{HostTrait, DeviceTrait, StreamTrait}; +use std::sync::{Arc, Mutex}; +use std::sync::mpsc; +use std::sync::mpsc::{Sender, Receiver}; +use rustfft::num_complex::Complex; +use std::thread; +use std::thread::sleep; +use std::time::Duration; +use hound::{ WavWriter, WavSpec, SampleFormat}; + +use crate::SPECTOGRAM_AREA; + +const DUMMY_READ_DELAY: u64 = 1; // seconds +const DUMMY_SAMPLE_SIZE: usize = 10; + +const DEFAULT_SAMPLE_RATE: u32 = 48_000; + +struct InputBuffer { + buffer: Arc> +} + +impl InputBuffer { + fn new(buffer: Arc>) -> Self { + Self { + buffer + } + } + + fn get_data(&mut self, data: &mut [i16]) { + let buffer = self.buffer.lock().unwrap(); + for (i, point) in data.iter_mut().enumerate() { + *point = buffer[i]; + } + } + + fn give_data(&mut self, data: &mut [i16]) { + let mut buffer = self.buffer.lock().unwrap(); + let length = data.len()/2; + + for (i, x) in (length..SPECTOGRAM_AREA).enumerate() { + buffer[i] = buffer[x]; + } + + for (i, x) in (SPECTOGRAM_AREA-length..SPECTOGRAM_AREA).enumerate() { + buffer[x] = data[i]; + } + } +} + +struct OutputBuffer { + buffer: Arc>, + index: usize, + tx: Sender, + rx: Receiver, +} + +impl OutputBuffer { + fn new(buffer: Arc>) -> Self { + let (tx, rx) = mpsc::channel(); + Self { + buffer, + index: 0, + tx, + rx + } + } + + fn get_data(&mut self, data: &mut [i16]) { + let mut buffer = self.buffer.lock().unwrap(); + let length = data.len()/2; + + for i in 0..length { + data[i*2] = buffer[i + self.index]; + } + + self.index += length; + if self.index > SPECTOGRAM_AREA { + for i in 0..SPECTOGRAM_AREA { + buffer[i] = buffer[i + SPECTOGRAM_AREA]; + } + self.index -= SPECTOGRAM_AREA; + + let _ = self.tx.send(true); + } + } + + fn give_data(&mut self, data: &[Complex]) { + if self.rx.recv().unwrap() { + let mut buffer = self.buffer.lock().unwrap(); + for (i, x) in buffer[SPECTOGRAM_AREA..].iter_mut().enumerate() { + *x = data[i].re as i16; + } + } + } +} + +enum Location { + Physical(StreamConfig), + File(String), + None +} + +pub struct SoundObject { + callback: fn(&mut [i16]), + input_buffer: InputBuffer, + output_buffer: OutputBuffer +} + +impl SoundObject { + fn new(input: Location, output: Location) -> Self { + let input_buffer_space = Arc::new(Mutex::new([0i16; SPECTOGRAM_AREA])); + let input_buffer = InputBuffer::new(input_buffer_space.clone()); + let thread_input_buffer = InputBuffer::new(input_buffer_space); + + let output_buffer_space = Arc::new(Mutex::new([0i16; 2 * SPECTOGRAM_AREA])); + let output_buffer = OutputBuffer::new(output_buffer_space.clone()); + let thread_output_buffer = OutputBuffer::new(output_buffer_space); + + let (sample_rate, callback):(u32, Box) = match input { + Location::Physical(config) => { + let host = cpal::default_host(); + let device = host.default_input_device().expect("No input device available"); + let stream = device.build_input_stream( + &config, + move | data: &mut [i16], _: &InputCallbackInfo | { + thread_input_buffer.give_data(data); + }, + move | err | { + eprintln!("an error occurred on the input audio stream: {}", err); + }, + None + ).unwrap(); + stream.play().expect("Output stream play failed"); + + let callback = Box::new(| data: &mut [i16] | { + input_buffer.get_data(data); + }); + (DEFAULT_SAMPLE_RATE, callback) + }, + Location::File(string) => { + let mut reader = hound::WavReader::open(string).unwrap(); + let sample_rate = reader.spec().sample_rate; + let mut samples = reader.samples(); + let callback = Box::new(| data: &mut [i16] | { + for sample in data.iter_mut() { + *sample = match samples.next() { + Some(x) => x.unwrap(), + None => { + samples = reader.samples(); + 0 + } + } as i16 + } + }); + (sample_rate, callback) + }, + Location::None => { + let callback = Box::new(| data: &mut [i16] | { + for point in data.iter_mut() { + *point = 0i16; + } + }); + (DEFAULT_SAMPLE_RATE, callback) + + } + }; + + match output { + Location::Physical(config) => { + let host = cpal::default_host(); + let device = host.default_output_device().expect("No output device available"); + let stream = device.build_output_stream( + &config, + move |data: &mut [i16], _: &OutputCallbackInfo| { + thread_output_buffer.get_data(data); + }, + move |err| { + eprintln!("an error occurred on the output audio stream: {}", err); + }, + None + ).unwrap(); + stream.play().expect("Output stream play failed"); + }, + Location::File(path) => { + let mut writer = WavWriter::create(path, WavSpec{ + channels: 1, + sample_rate, + bits_per_sample: 16, + sample_format: SampleFormat::Int + }).unwrap(); + let write_delay = DUMMY_SAMPLE_SIZE as f32 / sample_rate as f32; + let _ = thread::spawn(move || { + let mut data = [0i16; DUMMY_SAMPLE_SIZE]; + loop { + thread_output_buffer.get_data(&mut data[..]); + for i in 0..DUMMY_SAMPLE_SIZE { + writer.write_sample(data[i]).unwrap(); + } + sleep(Duration::from_secs_f32(write_delay)); + } + }); + }, + Location::None => { + let mut data = [0i16; DUMMY_SAMPLE_SIZE]; + let write_delay = DUMMY_SAMPLE_SIZE as f32 / sample_rate as f32; + let _ = thread::spawn(move || { + loop { + thread_output_buffer.get_data(&mut data[..]); + sleep(Duration::from_secs_f32(write_delay)); + } + }); + } + } + + Self { + callback, + input_buffer, + output_buffer + } + } +} + + -- 2.39.2