From 0488ca6e64a5524675b1fba5234dc1a15093786e Mon Sep 17 00:00:00 2001 From: Xaver Roos Date: Fri, 1 Apr 2022 16:14:28 +0200 Subject: [PATCH] [distance_calibration] sequence checkerboard detection --- etrack/__init__.py | 3 +- etrack/distance_calibration.py | 271 ++++++++++++++++++++++++--------- etrack/image_marker.py | 4 +- 3 files changed, 201 insertions(+), 77 deletions(-) diff --git a/etrack/__init__.py b/etrack/__init__.py index 486c4ba..a11f3ed 100644 --- a/etrack/__init__.py +++ b/etrack/__init__.py @@ -1,2 +1,3 @@ from .image_marker import ImageMarker, MarkerTask -from .tracking_result import TrackingResult \ No newline at end of file +from .tracking_result import TrackingResult +from .distance_calibration import DistanceCalibration \ No newline at end of file diff --git a/etrack/distance_calibration.py b/etrack/distance_calibration.py index c859b9e..2ab5363 100644 --- a/etrack/distance_calibration.py +++ b/etrack/distance_calibration.py @@ -1,17 +1,21 @@ -from cv2 import calibrationMatrixValues +from multiprocessing import allow_connection_pickling +from turtle import left +from cv2 import MARKER_TRIANGLE_UP, calibrationMatrixValues, threshold import matplotlib.pyplot as plt import numpy as np import cv2 import os import sys from IPython import embed +from etrack import MarkerTask, ImageMarker -class DistanceCalibration: +class DistanceCalibration(): - def __init__(self, file_name, x_0=95, y_0=185, cam_dist=1.36, width=1.35, height=0.805, width_pixel=1975, height_pixel=1375, checkerboard_width=0.24, checkerboard_height=0.18, + def __init__(self, file_name, frame_number, x_0=154, y_0=1318, cam_dist=1.36, width=1.35, height=0.805, width_pixel=1900, height_pixel=200, checkerboard_width=0.24, checkerboard_height=0.18, checkerboard_width_pixel=500, checkerboard_height_pixel=350, rectangle_width=0.024, rectangle_height=0.0225, rectangle_width_pixel=100, rectangle_height_pixel=90, - rectangle_count_width=9, rectangle_count_height=7) -> None: + rectangle_count_width=9, rectangle_count_height=7) -> None: + super().__init__() # aktualisieren """Calibration of the dimensions of the tank. Conversion of pixel into meter. Width refers to the "x-axis", height to the "y-axis" of the tank. @@ -21,9 +25,9 @@ class DistanceCalibration: y_0 (int, optional): Y-value of the "origin" of the tank. Defaults to 0. cam_dist (int, optional): Distance of camera lense to tank floor. Defaults to 1.36. width (int, optional): Width in meter from one lightened corner of the tank to the other. Defaults to 1.35. - heigth (int, optional): Height in meter from one lightened corner of the tank to the other. Defaults to 1.35. + height (int, optional): Height in meter from one lightened corner of the tank to the other. Defaults to 1.35. width_pixel (int, optional): Width in pixel from one lightened corner of the tank to the other. Defaults to 1975. - height_pixel (int, optional): Heigth in pixel from one lightened corner of the tank to the other. Defaults to 1375. + height_pixel (int, optional): Height in pixel from one lightened corner of the tank to the other. Defaults to 1375. rectangle_width (float, optional): Width of one black or corresponding white rectangle of the checkerboard. Defaults to 0.024. rectangle_height (float, optional): Height of one black or corresponding white rectangle of the checkerboard. Defaults to 0.0225. rectangle_count_width (int, optional): Number of black rectangles over the width of the whole checkerboard. Defaults to 9. @@ -47,15 +51,25 @@ class DistanceCalibration: self._x_factor = self.width / self.width_pix # m/pix self._y_factor = self.height / self.height_pix # m/pix - # properties + self.mark_crop_positions + self.threshold_crossings + @property def x_0(self): return self._x_0 + + @x_0.setter + def x_0(self, value): + self._x_0 = value @property def y_0(self): return self._y_0 + @y_0.setter + def y_0(self, value): + self._y_0 = value + @property def cam_dist(self): return self._cam_dist @@ -63,65 +77,140 @@ class DistanceCalibration: @property def width(self): return self._width - + + @width.setter + def width(self, value): + self._width = value + @property def height(self): return self._height + @height.setter + def height(self, value): + self._height = value + @property def width_pix(self): return self._width_pix + @width_pix.setter + def width_pix(self, value): + self._width_pix = value + @property def height_pix(self): return self._height_pix + @height_pix.setter + def height_pix(self, value): + self._height_pix_ = value + + @property + def cb_width(self): + return self._cb_width + + @cb_width.setter + def cb_width(self, value): + self._cb_width = value + + @property + def cb_height(self): + return self._cb_height + + @cb_height.setter + def cb_height(self, value): + self._cb_height = value + @property def x_factor(self): return self._x_factor @property def y_factor(self): - return self._y_factor + return self._y_factor - def crop_movie(): - if not os.path.exists(filename): - raise IOError("file %s does not exist!" % filename) - video = cv2.VideoCapture() - video.open(filename) - frame_counter = 0 - success = True - frame = None - while success and frame_counter <= frame_number: # iterating until frame_counter == frame_number --> success (True) - print("Reading frame: %i" % frame_counter, end="\r") - success, frame = video.read() - frame_counter += 1 - if success: - self._fig.gca().imshow(frame) # plot wanted frame of video - else: - print("Could not read frame number %i either failed to open movie or beyond maximum frame number!" % frame_number) - return [] - plt.ion() # turn on interactive mode - plt.show(block=False) # block=False allows to continue interact in terminal while the figure is open + def mark_crop_positions(self): + task = MarkerTask("crop area", ["bottom left corner", "top left corner", "top right corner", "bottom right corner"], "Mark crop area") + im = ImageMarker([task]) + + marker_positions = im.mark_movie(file_name, frame_number) + print(marker_positions) + + np.save('marker_positions', marker_positions) + + return marker_positions + + + def crop_frame(self, frame, marker_positions): + + bottom_left = marker_positions[0]['bottom left corner'] + bottom_right = marker_positions[0]['bottom right corner'] + top_left = marker_positions[0]['top left corner'] + top_right = marker_positions[0]['top right corner'] + + left_bound = int(np.mean([bottom_left[0], top_left[0]])) + right_bound = int(np.mean([bottom_right[0], top_right[0]])) + top_bound = int(np.mean([top_left[1], top_right[1]])) + bottom_bound = int(np.mean([bottom_left[1], bottom_right[1]])) + + crop_frame = frame[top_bound:bottom_bound, left_bound:right_bound] + crop_frame = np.mean(crop_frame, axis=2) + + frame_width = np.mean(crop_frame,axis=0) + frame_height = np.mean(crop_frame,axis=1) + + diff_width = np.diff(frame_width) + diff_height = np.diff(frame_height) + + x_width = np.arange(0, len(diff_width), 1) + x_height = np.arange(0, len(diff_height), 1) + return frame_width, frame_height, diff_width, diff_height, x_width, x_height + + + def rotation_angle(): + pass + + + def threshold_crossings(self, data, threshold_factor): + lower_threshold = np.min(data) / threshold_factor + upper_threshold = np.max(data) / threshold_factor - self._task_index = -1 - if len(self._tasks) > 0: - self._next_task() + lower_crossings = np.diff(data < lower_threshold, prepend=False) + upper_crossings = np.diff(data > upper_threshold, append=False) - while not self._tasks_done: - plt.pause(0.250) - if self._interrupt: - return [] + lower_crossings_indices = np.argwhere(lower_crossings) + upper_crossings_indices = np.argwhere(upper_crossings) + + half_window_size = 10 + lower_peaks = [] + upper_peaks = [] + for lower_idx in lower_crossings_indices: + if lower_idx < half_window_size: + half_window_size = lower_idx + window = data[lower_idx[0] - int(half_window_size):lower_idx[0] + int(half_window_size)] + min_window = np.min(window) + min_idx = np.where(data == min_window) + lower_peaks.append(min_idx) - self._fig.gca().set_title("All set and done!\n Window will close in 2s") - self._fig.canvas.draw() - plt.pause(2.0) - plt.close() - return [t.marker_positions for t in self._tasks] + for upper_idx in upper_crossings_indices: + if upper_idx < half_window_size: + half_window_size = upper_idx + window = data[upper_idx[0] - int(half_window_size) : upper_idx[0] + int(half_window_size)] + + max_window = np.max(window) + max_idx = np.where(data == max_window) + upper_peaks.append(max_idx) + + lower_peaks = np.unique(lower_peaks) + upper_peaks = np.unique(upper_peaks) + + return lower_peaks, upper_peaks - def mark_checkerboard(self, filename, frame_number=10): + def detect_checkerboard(self, filename, frame_number, marker_positions): + if not os.path.exists(filename): raise IOError("file %s does not exist!" % filename) video = cv2.VideoCapture() @@ -139,47 +228,81 @@ class DistanceCalibration: print("Reading frame: %i" % frame_counter, end="\r") success, frame = video.read() frame_counter += 1 - width_mean = np.mean(frame,axis=1) - crop_width_mean = width_mean[x_0:width_pix] - - height_mean = np.mean(frame,axis=0) - crop_height_mean = height_mean[y_0:height_pix] -# HELLO, here you at - embed() - quit() - if success: - self._fig.gca().imshow(frame) # plot wanted frame of video - else: - print("Could not read frame number %i either failed to open movie or beyond maximum frame number!" % frame_number) - return [] - plt.ion() # turn on interactive mode - plt.show(block=False) # block=False allows to continue interact in terminal while the figure is open + marker_positions = np.load('marker_positions.npy', allow_pickle=True) + + frame_width, frame_height, diff_width, diff_height, _, _ = dc.crop_frame(frame, marker_positions) - self._task_index = -1 - if len(self._tasks) > 0: - self._next_task() + # y-axis is inverted.. - while not self._tasks_done: - plt.pause(0.250) - if self._interrupt: - return [] + thresh_fact = 7 + lci_width, uci_width = dc.threshold_crossings(diff_width, threshold_factor=thresh_fact) + lci_height, uci_height = dc.threshold_crossings(diff_height, threshold_factor=thresh_fact) + + print('lower crossings:', lci_width) + print('upper crossings:', uci_width) - self._fig.gca().set_title("All set and done!\n Window will close in 2s") - self._fig.canvas.draw() - plt.pause(2.0) - plt.close() - return [t.marker_positions for t in self._tasks] + # make function for this + zip_list = [] + for zl in lci_width: + zip_list.append(zl) + for zu in uci_width: + zip_list.append(zu) + zip_list = np.sort(zip_list) -if __name__ == "__main__": - vid2 = "/home/efish/etrack/videos/2022.03.28_3.mp4" - calibration_task = DistanceCalibration(vid2) - dc = DistanceCalibration(calibration_task) - dc.mark_checkerboard(vid2, 10) + sequence = [] + for z in zip_list: + if z in lci_width: + sequence.append('down') + else: + sequence.append('up') + print('sequence:', sequence) + if sequence == ['up', 'down', 'up', 'down']: + print('in middle') + # first down, second up are edges of checkerboard + elif sequence == ['up', 'up', 'down']: + print('at left') + # first and second up are edges of checkerboard + else: + print('at right') + # first and second down are edges of checkerboard + + # find mistake in threshold detection (_7.mp4) where two detections at side (by thresh factor) + # find which indices (=pixels) represent edges of checkerboard by the corresponding sequence of ups and downs + # both for width and height + # assign x and y positions for the checkerboard corners + + # pixel to meter factor for default position with checkerboard in center of tank underneath camera + + plt.plot(diff_width) + plt.axhline(np.min(diff_width) / thresh_fact) + plt.axhline(np.max(diff_width) / thresh_fact) + for l in lci_width: + plt.axvline(l, color='yellow') + for u in uci_width: + plt.axvline(u, color='green') + # plt.plot(frame_height) + plt.plot(frame_width) + plt.show() + embed() + quit() + + # rotation angle + + +if __name__ == "__main__": + file_name = "/home/efish/etrack/videos/2022.03.28_7.mp4" + frame_number = 10 + dc = DistanceCalibration(file_name=file_name, frame_number=frame_number) + + # marker_positions = dc.mark_crop_positions() + + dc.detect_checkerboard(file_name, frame_number=frame_number, marker_positions=np.load('marker_positions.npy', allow_pickle=True)) + # print(sys.argv[0]) # print (sys.argv[1]) # vid1 = sys.argv[1] - embed() \ No newline at end of file + # embed() \ No newline at end of file diff --git a/etrack/image_marker.py b/etrack/image_marker.py index 82ca605..4fb6322 100644 --- a/etrack/image_marker.py +++ b/etrack/image_marker.py @@ -20,7 +20,7 @@ class ImageMarker: self._fig.canvas.mpl_connect('close_event', self._fig_close_event) self._fig.canvas.mpl_connect('key_press_event', self._key_press_event) - def mark_movie(self, filename, frame_number=0): + def mark_movie(self, filename, frame_number=10): """ Interactive GUI to mark the corners of the tank. A specific frame of the video can be chosen. Returns marker positions. Args: @@ -161,7 +161,7 @@ if __name__ == "__main__": im = ImageMarker(tasks) vid1 = "/home/efish/efish_tracking/efish_tracking3-Xaver-2022-03-21/videos/2022.01.12_3DLC_resnet50_efish_tracking3Mar21shuffle1_300000_labeled.mp4" - marker_positions = im.mark_movie(vid1, 100) + marker_positions = im.mark_movie(vid1, 10) print(marker_positions) # print(sys.argv[0])