From ff7cba862f5355bb9472fcba9cf1eabb5647b43a Mon Sep 17 00:00:00 2001 From: will Date: Sat, 2 Nov 2024 12:33:22 +0000 Subject: [PATCH] Begun setting up c++ functions - moved process of taking the photo to the rust programme with only the homography calculation and the actual perspective transformation as part of the cpp binary. currently having issues getting rust_cc to link the opencv library. --- Cargo.lock | 15 ++++++- Cargo.toml | 2 + src/build.rs | 24 ++++++++++-- src/camera.cpp | 10 ++--- src/main.rs | 59 +++++++++++++++++++++------- src/perspective.cpp | 95 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 src/perspective.cpp diff --git a/Cargo.lock b/Cargo.lock index ba08d99..86fccbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -851,9 +851,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libloading" @@ -1499,6 +1499,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" +[[package]] +name = "rscam" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89056084211cd54924fedf2e2199b906409d1f795cfd8e7e3271061742457018" +dependencies = [ + "libc", +] + [[package]] name = "rust_fft" version = "0.1.0" @@ -1506,6 +1515,8 @@ dependencies = [ "cc", "cpal", "hound", + "libc", + "rscam", "rustfft", "show-image", ] diff --git a/Cargo.toml b/Cargo.toml index 83fab88..54442bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,12 @@ build = "src/build.rs" include = ["src/camera.cpp"] [dependencies] +libc = "0.2.161" rustfft = "6.2.0" show-image = "0.14.0" hound = "3.5.1" cpal = "0.15.3" +rscam = "0.5.5" [build-dependencies] cc = "1.0" diff --git a/src/build.rs b/src/build.rs index 0f5bf50..580e1f5 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1,7 +1,25 @@ fn main() { + println!("cargo:rerun-if-changed=src/perspective.cpp"); + println!("cargo::rustc-link-lib=dylib=opencv_core"); + println!("cargo::rustc-link-lib=dylib=opencv_highgui"); + println!("cargo::rustc-link-lib=dylib=opencv_xfeatures2d"); + println!("cargo::rustc-link-lib=dylib=opencv_calib3d"); + println!("cargo::rustc-link-lib=dylib=opencv_videoio"); + println!("cargo::rustc-link-lib=dylib=opencv_imgcodecs"); + println!("cargo::rustc-link-lib=dylib=opencv_features2d"); + + let opencv_path = std::path::Path::new("/usr/share/include/opencv4/"); cc::Build::new() .cpp(true) - .file("src/camera.cpp") - .compile("camera.a"); - println!("cargo:rerun-if-changed=src/camera.cpp"); + .include(opencv_path) + .flag("-Wunused-parameter") + .flag("-lopencv_core") + .flag("-lopencv_highgui") + .flag("-lopencv_xfeatures2d") + .flag("-lopencv_calib3d") + .flag("-lopencv_videoio") + .flag("-lopencv_imgcodecs") + .flag("-lopencv_features2d") + .file("src/perspective.cpp") + .compile("perspective.a"); } diff --git a/src/camera.cpp b/src/camera.cpp index 74233cf..386b584 100644 --- a/src/camera.cpp +++ b/src/camera.cpp @@ -9,13 +9,11 @@ const size_t CHUNK_SIZE = 72; const size_t SPECTOGRAM_AREA = WINDOW_SIZE * CHUNK_SIZE; extern "C" { - uint8_t GetData(size_t *ptr) { + void GetData(uint8_t *ptr) { + uint8_t image[SPECTOGRAM_AREA * 3]; - for (uint8_t i : image) { - i = *ptr; - ptr++; + for (long unsigned int i = 0; i < SPECTOGRAM_AREA * 3; i++) { + image[i] = *(ptr + i); } - uint8_t value = image[1]; - return value; } } diff --git a/src/main.rs b/src/main.rs index f9e048c..c1b5207 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,11 +8,17 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use std::sync::{Arc, Mutex}; use std::sync::mpsc; use std::sync::mpsc::Sender; +use rscam::{Camera, Config}; const WINDOW_SIZE: usize = 128; const CHUNK_SIZE: usize = 72; const SPECTOGRAM_AREA: usize = WINDOW_SIZE * CHUNK_SIZE; +const IMAGE_WIDTH: usize = 1920; +const IMAGE_HEIGHT: usize = 1080; +const IMAGE_AREA: usize = IMAGE_WIDTH * IMAGE_HEIGHT * 3; +const FPS: usize = 30; + const AMPLITUDE_MAX: f32 = 255.0; const AMPLITUDE_MIN: f32 = 0.0; const AMPLITUDE_REL: f32 = AMPLITUDE_MAX - AMPLITUDE_MIN; @@ -25,17 +31,50 @@ const VOLUME_MAX: f32 = 100.0; // 60 - 65 const VOLUME_MIN: f32 = -40.0; const VOLUME_REL: f32 = VOLUME_MAX - VOLUME_MIN; +extern "C" { + fn GetHomography(camera_ptr: usize, homography_ptr: usize); + fn ApplyHomography(camera_ptr: usize, buffer_ptr: usize, homography_ptr: usize); +} + struct ImageArray { data: [u8; SPECTOGRAM_AREA * 3], + homography: [f64; 9], + camera_buffer: [u8; IMAGE_AREA], + camera: Camera, chunks: usize } impl ImageArray { - fn new () -> Self { - Self { + fn new (homography: [f64; 9]) -> Self { + let mut array = Self { data: [0u8; SPECTOGRAM_AREA * 3], + homography, + camera_buffer: [0u8; IMAGE_AREA], + + camera: Camera::new("/dev/video0").unwrap(), + chunks: SPECTOGRAM_AREA + }; + array.camera.start(&Config { + interval: (1, FPS as u32), + resolution: (IMAGE_WIDTH as u32, IMAGE_HEIGHT as u32), + format: b"RGB3", + ..Default::default() + }).unwrap(); + array + } + + fn from_camera (&mut self) { + self.camera_buffer = self.camera.capture().unwrap()[..].try_into().expect("Image is wrong size"); + + unsafe{ApplyHomography (self.camera_buffer.as_ptr() as usize, self.data.as_ptr() as usize, self.homography.as_ptr() as usize);} + } + + fn calibrate (&mut self) { + unsafe { + GetHomography(self.camera_buffer.as_ptr() as usize, self.homography.as_ptr() as usize); } + println!("Homography: {:?}", self.homography) } fn from_buffer (&mut self, buffer: &Vec>) -> () { @@ -129,10 +168,6 @@ impl SampleBuffer { } } -extern "C" { - fn GetData(ptr: usize) -> usize; -} - #[show_image::main] fn main () -> Result<(), Box> { @@ -147,10 +182,12 @@ fn main () -> Result<(), Box> { let sample_buffer = Arc::new(Mutex::new([0i16; 2*SPECTOGRAM_AREA])); let mut samples = SampleBuffer::new(sample_buffer.clone(), tx); - let mut image_array = ImageArray::new(); - let window = create_window("image", Default::default())?; + let homography = [0f64; 9]; + let mut image_array = ImageArray::new(homography); + image_array.calibrate(); + let mut reader = hound::WavReader::open("/home/will/Downloads/Adducci - Around the Horn.wav").unwrap(); let file_rate = reader.spec().sample_rate; @@ -186,11 +223,7 @@ fn main () -> Result<(), Box> { let image = ImageView::new(ImageInfo::rgb8(WINDOW_SIZE as u32, CHUNK_SIZE as u32), &image_array.data); window.set_image ("image", image)?; - unsafe{ - println!("First value: {}", image_array.data[1]); - println!("Origional: {}", image_array.data.as_ptr() as usize); - println!("Read first value: {}", GetData(image_array.data.as_ptr() as usize)); - } + image_array.from_camera(); image_array.to_buffer(&mut buffer); diff --git a/src/perspective.cpp b/src/perspective.cpp new file mode 100644 index 0000000..dce16a6 --- /dev/null +++ b/src/perspective.cpp @@ -0,0 +1,95 @@ +// g++ ./perspective.cpp -I/usr/share/include/opencv4/ -lopencv_core -lopencv_calib3d -lopencv_highgui -lopencv_xfeatures2d -lopencv_features2d -lopencv_imgproc -lopencv_videoio -lopencv_imgcodecs -lopencv_features2d -o perspective.a +// +#include "opencv2/core.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/xfeatures2d.hpp" +#include "opencv2/calib3d.hpp" + +using namespace cv; +using namespace cv::xfeatures2d; + +const size_t WINDOW_SIZE = 128; +const size_t CHUNK_SIZE = 72; +const size_t SPECTOGRAM_AREA = WINDOW_SIZE * CHUNK_SIZE; +const Size DSIZE_AREA = Size(WINDOW_SIZE, CHUNK_SIZE); + +// Mat get_frame() +// { +// VideoCapture cap; +// Mat frame; +// +// int device = 0; +// int api = CAP_V4L; +// cap.open(device, api); +// +// cap.read(frame); +// resize(frame, frame, DSIZE_AREA, INTER_LINEAR); +// return frame; +// } + +int main() +{ + std::cout << "all normal"; + return 0; +} + +extern "C" +{ + void GetHomography(uint8_t *camera_ptr, double *homography_ptr) + { + Mat img1 = imread( samples::findFile("box.jpg"), IMREAD_GRAYSCALE ); + Mat img2 = imread( samples::findFile("box_in_scene.jpg"), IMREAD_GRAYSCALE ); + + // detect keypoints and compute descriptors + int minHessian = 400; + Ptr detector = SURF::create( minHessian ); + + std::vector keypoints1, keypoints2; + Mat descriptors1, descriptors2; + detector->detectAndCompute( img1, noArray(), keypoints1, descriptors1 ); + detector->detectAndCompute( img2, noArray(), keypoints2, descriptors2 ); + + // match descriptors + Ptr matcher = DescriptorMatcher::create(DescriptorMatcher::FLANNBASED); + std::vector< std::vector > knn_matches; + + matcher->knnMatch( descriptors1, descriptors2, knn_matches, 2 ); + + // filter matches by the ratio test + const float ratio_thresh = 0.7f; + std::vector good_matches; + for (size_t i = 0; i < knn_matches.size(); i++) + { + if (knn_matches[i][0].distance < ratio_thresh * knn_matches[i][1].distance) + { + good_matches.push_back(knn_matches[i][0]); + } + } + + // get the source and destination points + std::vector source_points, dst_points; + for (size_t i = 0; i < good_matches.size(); i++) + { + Point2f s_point = keypoints2[good_matches[i].trainIdx].pt; + Point2f d_point = keypoints1[good_matches[i].queryIdx].pt; + source_points.push_back(s_point); + dst_points.push_back(d_point); + } + + // perform homography + double ransac_thresh = 5.0f; + Mat homography = findHomography(source_points, dst_points, RANSAC, ransac_thresh); + + const double* h_ptr = homography.ptr(0); + for(int i = 0; i < 9; i++) + *homography_ptr = std::max(h_ptr[i], 0.); + } + + void ApplyHomography(uint8_t *camera_ptr, uint8_t *buffer_ptr, float *homography_ptr) + { + Mat output_image; + // warpPerspective(img2, output_image, homography, img1.size(), INTER_NEAREST); + // imwrite("output.jpg", output_image ); + } +} + -- 2.39.2