From: will Date: Sat, 24 Aug 2024 15:52:37 +0000 (+0100) Subject: Fixed lookup table system X-Git-Url: https://git.ozva.co.uk/?a=commitdiff_plain;h=63d008658815ff6fed5c3cb609d9c7a0c1af740c;p=audio-over-stft Fixed lookup table system Currently untested, possibly works? --- diff --git a/__pycache__/camera.cpython-311.pyc b/__pycache__/camera.cpython-311.pyc index b6d39f2..60c2e46 100644 Binary files a/__pycache__/camera.cpython-311.pyc and b/__pycache__/camera.cpython-311.pyc differ diff --git a/__pycache__/fft.cpython-311.pyc b/__pycache__/fft.cpython-311.pyc index a73cf1d..a40f768 100644 Binary files a/__pycache__/fft.cpython-311.pyc and b/__pycache__/fft.cpython-311.pyc differ diff --git a/camera.py b/camera.py index 62efb3f..2539baf 100644 --- a/camera.py +++ b/camera.py @@ -1,13 +1,16 @@ import cv2 as cv import numpy as np import queue +import time import threading import random +import matplotlib.pyplot as plt class VideoCapture: def __init__(self, device_id, backend): self.camera = cv.VideoCapture(device_id, backend) + self.camera.set(cv.CAP_PROP_BUFFERSIZE, 1) self.camera.set(cv.CAP_PROP_FRAME_WIDTH, 1920.0) self.camera.set(cv.CAP_PROP_FRAME_HEIGHT, 1080.0) @@ -44,6 +47,7 @@ class camera(): device_id: int = 0, debug: bool = True, dummy: bool = False, + use_lookup: bool = False, use_files: bool = False ): @@ -53,17 +57,19 @@ class camera(): self.match_histograms = False self.show_debug = debug self.dummy = dummy + self.use_lookup = use_lookup + self.use_files = use_files - self.camera = VideoCapture(device_id, cv.CAP_V4L) + self.camera = cv.VideoCapture(device_id, cv.CAP_V4L) - self.camera.set(cv.CAP_PROP_BUFFERSIZE, 38) + self.camera.set(cv.CAP_PROP_FPS, 30) + self.camera.set(cv.CAP_PROP_BUFFERSIZE, 1) self.camera.set(cv.CAP_PROP_FRAME_WIDTH, 1920.0) self.camera.set(cv.CAP_PROP_FRAME_HEIGHT, 1080.0) self.homography = None - self.lookup_color = None - self.lookup_vingette = None - self.lookup_compression = None + self.lookup = None + self.lookup_compression = 100 self.last_display = None self.last_capture = None self.last_recovered = None @@ -110,135 +116,50 @@ class camera(): else: print("calibration failed") + quit() + + if self.use_file == True: + self.get_lookup() def get_lookup( self ) -> None: - if self.dummy == True: - pass #return - - vingette_compression = 50 - - self.lookup_vingette = np.zeros(( - 255 // vingette_compression + 1, - self.display_size[0] // vingette_compression + 1, - self.display_size[1] // vingette_compression + 1 - ), dtype=np.uint8) - - for v in range(0, 255, vingette_compression): - pixel = np.array([[[0, 0, v]]], dtype=np.uint8) - pixel = cv.cvtColor(pixel, cv.COLOR_HSV2BGR) - - self.display(pixel) - #capture = self.capture() - #capture = cv.cvtColor(capture, cv.COLOR_BGR2HSV) - - # apply vingette overlay - capture = cv.resize(pixel, self.display_size, interpolation=cv.INTER_NEAREST_EXACT) - - rows = len(capture) - columns = len(capture[0]) - for y, row in enumerate(capture): - for x, pixel in enumerate(row): - y -= rows // 2 - x -= columns // 2 - math_pixel = pixel.astype(np.int16) - if x > y: - math_pixel += round(y * 0.0043) - else: - math_pixel += round(x * 0.0043) - - pixel = np.clip(math_pixel, 0, 255).astype(np.uint8) - - - for y in range(0, self.display_size[0], vingette_compression): - for x in range(0, self.display_size[1], vingette_compression): - self.lookup_vingette[v // vingette_compression, y // vingette_compression, x // vingette_compression] = capture[y, x, 2] - v - - color_compression = 90 + if self.use_lookup == True: + return - self.lookup_color = np.zeros(( - 180 // color_compression + 1, - 255 // color_compression + 1, - 255 // color_compression + 1, - 3 - ), dtype=np.uint8) + if self.use_files == True: + self.lookup = np.load("lookup.npy") + return - for h in range(0, 180, color_compression): - for s in range(0, 255, color_compression): - for v in range(0, 255, color_compression): - pixel = np.array([[[h, s, v]]], dtype=np.uint8) - pixel = cv.cvtColor(pixel, cv.COLOR_HSV2BGR) + lookup = None + for r in range(0, 255, self.lookup_compression): + for g in range(0, 255, self.lookup_compression): + for b in range(0, 255, self.lookup_compression): + pixel = np.array([[[b, g, r]]], dtype=np.int8) + pixel = cv.resize(pixel, self.display_size, interpolation=cv.INTER_NEAREST_EXACT) self.display(pixel) - #capture = self.capture() - #capture = cv.cvtColor(capture, cv.COLOR_BGR2HSV) - - # apply the color ovelay - pixel = np.sin(pixel / (127.5 / np.pi)) * 2 - pixel = np.clip(pixel, 0, 255) - capture = cv.resize(pixel, self.display_size, interpolation=cv.INTER_NEAREST_EXACT) - - cx = self.display_size[1] // 2 - cy = self.display_size[0] // 2 - sample_diameter = 5 - color = np.average(capture[cy - sample_diameter:cy + sample_diameter, cx - sample_diameter:cx + sample_diameter], axis=(0,1)) - - self.lookup_color[h // color_compression, s // color_compression, v // color_compression, 0:2] = color - [h, s, v] - - np.save("lookup_vingette", self.lookup_vingette) - np.save("lookup_color", self.lookup_color) - def apply_lookup( - self, - image: np.ndarray - ) -> np.ndarray: - # must be given an array in the HSV color space + for i in range(10): # silly hack + time.sleep(0.2) + recovered = self.capture() - # apply the vingentte and color lookup table - for y, row in enumerate(image): - for x, pixel in enumerate(row): - pixel[2] += self.lookup_vingette[pixel[2] // self.lookup_compression][y // self.lookup_compression][x // self.lookup_compression] - pixel = self.lookup_color[pixel[0] // self.lookup_compression][pixel[1] // self.lookup_compression][pixel[2] // self.lookup_compression] + error = np.copy(recovered.astype(np.int16)) + error[..., 0] -= b + error[..., 1] -= g + error[..., 2] -= r - return image - - def lookup_debug( - self, - tests: int - ) -> np.ndarray: + error = np.clip(error, -50, 255) - error_array = None - - # begin loop over the desired sample size - for i in range(tests): - pixel = [[[random.randint(0,255), random.randint(0,255), random.randint(0,255)]]] - - # apply the color ovelay - pixel = np.sin(pixel / (127.5 / np.pi)) * 2 - pixel = np.clip(pixel, 0, 255) - - capture = cv.resize(pixel, self.display_size, interpolation=cv.INTER_NEAREST_EXACT) - rows = len(capture) - columns = len(capture[0]) - for y, row in enumerate(capture): - for x, pixel in enumerate(y): - y -= rows - x -= columns - distance = np.sqrt((x ^ 2) + (y ^ 2)) * 0.02 - pixel += distance - capture = np.clip(capture, 0, 255) - - returned = self.apply_lookup(capture) - self.display(returned) - - if error_array is None: - error_array = error - error_array = (error_array + error) / 2 + if lookup is None: + lookup = error + else: + lookup += error + lookup = lookup // 2 - error_array = cv.cvtColor(error_array, cv.COLOR_BGR2HSV) - cv.imwrite("error_array.png", error_array) + self.lookup = lookup + np.save("lookup.npy", lookup) def display( self, @@ -278,8 +199,8 @@ class camera(): image = cv.warpPerspective(image, self.homography, self.display_size) image = cv.resize(image, (self.window_size, self.window_height)) - if self.lookup_vingette is not None and self.lookup_color is not None: - pass + if type(self.lookup) == np.ndarray and self.use_lookup == True: + image = np.clip(image + self.lookup, 0, 255).astype(np.uint8) self.last_recovered = image diff --git a/fft.py b/fft.py index 876c1db..c74c4dc 100644 --- a/fft.py +++ b/fft.py @@ -16,7 +16,7 @@ class fft(): self.amplitude_max = 254 self.amplitude_min = 0 - self.angle_max = 179 + self.angle_max = 254 self.angle_min = 0 self.amplitude_relative = self.amplitude_max - self.amplitude_min @@ -42,9 +42,9 @@ class fft(): angle = ((angle + np.pi) * (self.angle_relative / (2 * np.pi))) + self.angle_min angle = np.clip(angle, self.angle_min, self.angle_max) - full = np.full(angle.shape, fill_value=255) + full = np.full(angle.shape, fill_value=60) - image = np.stack((angle, full, amplitude), axis=-1) + image = np.stack((full, angle, amplitude), axis=-1) image = np.array([image], dtype=np.uint8) image = cv.cvtColor(image, cv.COLOR_HSV2BGR) @@ -59,7 +59,7 @@ class fft(): image = cv.cvtColor(image, cv.COLOR_BGR2HSV) amplitude = image[0][...,2].astype(np.float64) - angle = image[0][...,0].astype(np.float64) + angle = image[0][...,1].astype(np.float64) amplitude -= self.amplitude_min amplitude /= (self.amplitude_relative / self.upper_limit) diff --git a/loop.py b/loop.py index 227c078..90c5e27 100644 --- a/loop.py +++ b/loop.py @@ -22,35 +22,32 @@ notes: some way of arranging the pixles independant of the orrigional spectrogram """ -sample_rate, data = wavfile.read("/home/will/Downloads/Adducci - Around the Horn.wav") +sample_rate, data = wavfile.read("/home/will/Downloads/birdsong.wav") #data = data[...,0] -new_rate = 11025. +new_rate = 10000. sample_count = round(len(data) * new_rate / sample_rate) data = sps.resample(data, sample_count) sample_rate = int(new_rate) -window_size = 192 -window_height = 108 +window_size = 130 +window_height = 70 hop_size = window_size // 2 camera = camera( window_size, window_height, (1840, 1000), - device_id = 0, - debug = True, + device_id = 2, + debug = False, dummy = False, - use_files = False + use_lookup = False, + use_files = True ) camera.calibrate() -# camera.get_lookup() -# print(camera.lookup_vingette) -# print(camera.lookup_color) -# lookup_debug(100) -# quit() +camera.get_lookup() transform = fft(window_size, hop_size) @@ -80,7 +77,7 @@ stream = pyaudio_object.open( format = pyaudio.paInt16, channels = 1, rate = sample_rate, - frames_per_buffer = 2048, + frames_per_buffer = 1024, output = True, stream_callback = callback ) @@ -117,7 +114,7 @@ try: if segment_index == segment_count: segment_index = 0 slept = 0 - while len(audio) > 3 * segment_samples: + while len(audio) > 5 * segment_samples: time.sleep(0.01) slept += 1 print(f"slept {slept} times")