[classifier] working but not really...

fixtracks/utils/signals.py

@@ -22,6 +22,7 @@ class DetectionSceneSignals(QObject):
 
 class DetectionTimelineSignals(QObject):
     windowMoved = Signal()
+    manualMove = Signal()
 
 class DetectionSignals(QObject):
     hover = Signal((int, QPointF))

fixtracks/utils/trackingdata.py

@@ -57,7 +57,7 @@ class TrackingData(QObject):
         return self._selection
 
     def setSelectionRange(self, col, start, stop):
-        logging.debug("Trackingdata: set selection range based on column %s to %.2f - %.2f", col, start, stop)
+        logging.info("Trackingdata: set selection range based on column %s to %.2f - %.2f", col, start, stop)
         col_indices = np.where((self[col] >= start) & (self[col] < stop))[0]
         self._selection = self._indices[col_indices]
         if len(col_indices) < 1:
@@ -77,10 +77,9 @@ class TrackingData(QObject):
         ids : array-like
             An array-like object containing the IDs to be set as user selections. 
         """
-        print(ids)
+        logging.debug("TrackingData.setSelection: %i number of ids", len(ids))
         self._selection = self._find(ids)
         self._selected_ids = ids
-        print(self._selection, self._selected_ids)
 
     def setTrack(self, track_id:int, setUserLabeled:bool=True)-> None:
         """Assign a new track_id to the user-selected detections
@@ -92,7 +91,7 @@ class TrackingData(QObject):
         setUserLabeled : bool
             Should the "userlabeled" state of the detections be set to True? Otherwise they will be left untouched.
         """
-        print(self._selection)
+        logging.info("TrackingData: set track id %i for selection, set user-labeled status %s", track_id, str(setUserLabeled))
         self["track"][self._selection] = track_id
         if setUserLabeled:
             self.setUserLabeledStatus(True, True)
@@ -114,7 +113,7 @@ class TrackingData(QObject):
         else:
             self["userlabeled"][:] = new_status
 
-    def revertAssignmentStatus(self):
+    def revertUserLabeledStatus(self):
         logging.debug("TrackingData:Un-setting assignment status of all data!")
         self["userlabeled"][:] = False
 

fixtracks/widgets/classifier.py

@@ -12,6 +12,17 @@ from fixtracks.utils.trackingdata import TrackingData
 
 from IPython import embed
 
+class Detection():
+    def __init__(self, id, frame, track, position, orientation, length, userlabeled):
+        self.id = id
+        self.frame = frame
+        self.track = track
+        self.position = position
+        self.score = 0.0
+        self.angle = orientation
+        self.length = length
+        self.userlabeled = userlabeled
+
 class WorkerSignals(QObject):
     error = Signal(str)
     running = Signal(bool)
@@ -24,7 +35,8 @@ class ConsitencyDataLoader(QRunnable):
         super().__init__()
         self.signals = WorkerSignals()
         self.data = data
-        self.bendedness = self.positions = None
+        self.bendedness = None
+        self.positions = None
         self.lengths = None
         self.orientations = None
         self.userlabeled = None
@@ -70,6 +82,18 @@ class ConsistencyWorker(QRunnable):
 
     @Slot()
     def run(self):
+        def get_detections(frame, indices):
+            detections = []
+            for i in indices:
+                if np.any(self.positions[i] < 0.1):
+                    logging.debug("Encountered probably invalid position %s", str(self.positions[i]))
+                    continue
+                d = Detection(i, frame, self.tracks[i], self.positions[i],
+                              self.orientations[i], self.lengths[i],
+                              self.userlabeled[i])
+                detections.append(d)
+            return detections
+
         def needs_checking(original, new):
             res = False
             for n, o in zip(new, original):
@@ -82,107 +106,135 @@ class ConsistencyWorker(QRunnable):
                 print("all detections would be assigned to one track!")
             return res
 
-        def assign_by_distance(f, p):
+        def assign_by_distance(d):
-            t1_step = f - last_frame[0]
+            t1_step = d.frame - last_detections[1].frame
-            t2_step = f - last_frame[1]
+            t2_step = d.frame - last_detections[2].frame
             if t1_step == 0 or t2_step == 0:
-                print(f"framecount is zero! current frame {f}, last frame {last_frame[0]} and {last_frame[1]}")
+                print(f"framecount is zero! current frame {f}, last frame {last_detections[1].frame} and {last_detections[2].frame}")
-
+            distance_to_trackone = np.linalg.norm(d.position - last_detections[1].position)/t1_step
-            distance_to_trackone = np.linalg.norm(p - last_pos[0])/t1_step
+            distance_to_tracktwo = np.linalg.norm(d.position - last_detections[2].position)/t2_step
-            distance_to_tracktwo = np.linalg.norm(p - last_pos[1])/t2_step
             most_likely_track = np.argmin([distance_to_trackone, distance_to_tracktwo]) + 1
             distances = np.zeros(2)
             distances[0] = distance_to_trackone
             distances[1] = distance_to_tracktwo
             return most_likely_track, distances
 
-        def assign_by_orientation(f, o):
+        def assign_by_orientation(d):
-            t1_step = f - last_frame[0]
+            t1_step = d.frame - last_detections[1].frame
-            t2_step = f - last_frame[1]
+            t2_step = d.frame - last_detections[2].frame
-            orientationchange = (last_angle - o)
+            orientationchanges = np.zeros(2)
-            orientationchange[orientationchange > 180] = 360 - orientationchange[orientationchange > 180]
+            for i in [1, 2]:
-            orientationchange /= np.array([t1_step, t2_step])
+                orientationchanges[i-1] = (last_detections[i].angle - d.angle)
-            # orientationchange = np.abs(np.unwrap((last_angle - o)/np.array([t1_step, t2_step])))
+
-            most_likely_track = np.argmin(np.abs(orientationchange)) + 1
+            orientationchanges[orientationchanges > 180] = 360 - orientationchanges[orientationchanges > 180]
-            return most_likely_track, orientationchange
+            orientationchanges /= np.array([t1_step, t2_step])
-
+            most_likely_track = np.argmin(np.abs(orientationchanges)) + 1
-        def assign_by_length(o):
+            return most_likely_track, orientationchanges
-            length_difference = np.abs((last_length - o))
+
-            most_likely_track = np.argmin(length_difference) + 1
+        def assign_by_length(d):
-            return most_likely_track, length_difference
+            length_differences = np.zeros(2)
-
+            length_differences[0] = np.abs((last_detections[1].length - d.length))
-        def do_assignment(f, indices, assignments):
+            length_differences[1] = np.abs((last_detections[2].length - d.length))
-            for i, idx in enumerate(indices):
+            most_likely_track = np.argmin(length_differences) + 1
-                self.tracks[idx] = assignments[i]
+            return most_likely_track, length_differences
-                last_pos[assignments[i]-1] = pp[i]
+
-                last_frame[assignments[i]-1] = f
+        unique_frames = np.unique(self.frames)
-                last_angle[assignments[i]-1] = self.orientations[idx]
+        steps = int((len(unique_frames) - self._startframe) // 100)
-                last_length[assignments[i]-1] += ((self.lengths[idx] - last_length[assignments[i]-1])/processed)
-
-        last_pos = [self.positions[(self.tracks == 1) & (self.frames <= self._startframe)][-1],
-                    self.positions[(self.tracks == 2) & (self.frames <= self._startframe)][-1]]
-        last_frame = [self.frames[(self.tracks == 1) & (self.frames <= self._startframe)][-1],
-                      self.frames[(self.tracks == 2) & (self.frames <= self._startframe)][-1]]
-        last_angle = [self.orientations[(self.tracks == 1) & (self.frames <= self._startframe)][-1],
-                      self.orientations[(self.tracks == 2) & (self.frames <= self._startframe)][-1]]
-        last_length = [self.lengths[(self.tracks == 1) & (self.frames <= self._startframe)][-1],
-                       self.lengths[(self.tracks == 2) & (self.frames <= self._startframe)][-1]]
-
         errors = 0
         processed = 1
         progress = 0
         self._stoprequest = False
-        maxframes = np.max(self.frames)
+        last_detections = {1: None, 2: None, -1: None}
-        startframe = np.max(last_frame)
-        steps = int((maxframes - startframe) // 200)
 
-        for f in np.unique(self.frames[self.frames > startframe]):
+        for f in unique_frames[unique_frames >= self._startframe]:
-            processed += 1
-            self.signals.currentframe.emit(f)
             if self._stoprequest:
                 break
+            error = False
+            self.signals.currentframe.emit(f)
             indices = np.where(self.frames == f)[0]
-            pp  = self.positions[indices]
+            detections = get_detections(f, indices)
-            originaltracks = self.tracks[indices]
+            done = [False, False]
-            dist_assignments = np.zeros_like(originaltracks)
+            if len(detections) == 0:
-            angle_assignments = np.zeros_like(originaltracks)
+                continue
-            length_assignments = np.zeros_like(originaltracks)
+
-            userlabeled = np.zeros_like(originaltracks)
+            if len(detections) > 1 and np.any([detections[0].userlabeled, detections[1].userlabeled]):
-            distances = np.zeros((len(originaltracks), 2))
+                # more than one detection
+                if detections[0].userlabeled and detections[1].userlabeled:
+                    if detections[0].track == detections[1].track:
+                        error = True
+                        logging.info("Classification error both detections in the same frame are assigned to the same track!")
+                elif detections[0].userlabeled and not detections[1].userlabeled:
+                    detections[1].track = 1 if detections[0].track == 2 else 2
+                elif not detections[0].userlabeled and detections[1].userlabeled:
+                    detections[0].track = 1 if detections[1].track == 2 else 2
+
+                if not error:
+                    last_detections[detections[0].track] = detections[0]
+                    last_detections[detections[1].track] = detections[1]
+                    self.tracks[detections[0].id] = detections[0].track
+                    self.tracks[detections[1].id] = detections[1].track
+                    done[0] = True
+                    done[1] = True
+            elif len(detections) == 1 and detections[0].userlabeled:  # ony one detection and labeled
+                last_detections[detections[0].track] = detections[0]
+                done[0] = True
+
+            if np.sum(done) == len(detections):
+                continue
+            # if f == 2088:
+            #     embed()
+            #     return
+            if error and self._stoponerror:
+                self.signals.error.emit("Classification error both detections in the same frame are assigned to the same track!")
+                break
+            dist_assignments = np.zeros(2, dtype=int)
+            orientation_assignments = np.zeros_like(dist_assignments)
+            length_assignments = np.zeros_like(dist_assignments)
+            distances = np.zeros((2, 2))
             orientations = np.zeros_like(distances)
             lengths = np.zeros_like(distances)
+            assignments = np.zeros((2, 2))
+            for i, d in enumerate(detections):
+                dist_assignments[i], distances[i, :] = assign_by_distance(d)
+                orientation_assignments[i], orientations[i,:] = assign_by_orientation(d)
+                length_assignments[i], lengths[i, :] = assign_by_length(d)
+                assignments[i, :] = dist_assignments  # (dist_assignments * 10 + orientation_assignments + length_assignments) / 3
+
+            diffs = np.diff(assignments, axis=1)
+            error = False
+            temp = {}
+            message = ""
+            for i, d in enumerate(detections):
+                temp = {}
+                if diffs[i] == 0:  # both are equally likely
+                    d.track = -1
+                    error = True
+                    message = "Classification error both detections in the same frame are assigned to the same track!"
+                    break
+                if diffs[i] < 0:
+                    d.track = 1
+                else:
+                    d.track = 2
+                self.tracks[d.id] = d.track
+                if d.track not in temp:
+                    temp[d.track] = d
+                else:
+                    error = True
+                    message = "Double assignment to the same track!"
+                    break
 
-            for i, (idx, p) in enumerate(zip(indices, pp)):
+            if not error:
-                if self.userlabeled[idx]:
+                for k in temp:
-                    print("user")
+                    last_detections[temp[k].track] = temp[k]
-                    userlabeled[i] = True
-                    last_pos[originaltracks[i]-1] = pp[i]
-                    last_frame[originaltracks[i]-1] = f
-                    last_angle[originaltracks[i]-1] = self.orientations[idx]
-                    last_length[originaltracks[i]-1] += ((self.lengths[idx] - last_length[originaltracks[i]-1]) / processed)
-                    continue
-                dist_assignments[i], distances[i, :] = assign_by_distance(f, p)
-                angle_assignments[i], orientations[i,:] = assign_by_orientation(f, self.orientations[idx])
-                length_assignments[i], lengths[i, :] = assign_by_length(self.lengths[idx])
-            if np.any(userlabeled):
-                continue
-            # check (re) assignment, update, and proceed
-            if not needs_checking(originaltracks, dist_assignments):
-                do_assignment(f, indices, dist_assignments)
             else:
-                if not (np.all(length_assignments == 1) or np.all(length_assignments == 2)):  # if I find a solution by body length
+                logging.info("frame %i: Cannot decide who is who! %s", f, message)
-                    logging.debug("frame %i: Decision based on body length", f)
+                for idx in indices:
-                    do_assignment(f, indices, length_assignments)
+                    self.tracks[idx] = -1
-                elif not (np.all(angle_assignments == 1) or np.all(angle_assignments == 2)):  # else there is a solution based on orientation
+                errors += 1
-                    logging.info("frame %i: Decision based on orientation", f)
+                if self._stoponerror:
-                    do_assignment(f, indices, angle_assignments)
+                    self.signals.error.emit(message)
-                else:
+                    break
-                    logging.info("frame %i: Cannot decide who is who")
+            processed += 1
-                    for idx in indices:
-                        self.tracks[idx] = -1
-                    errors += 1
-                    if self._stoponerror:
-                        break
 
             if steps > 0 and f % steps == 0:
                 progress += 1
@@ -481,18 +533,25 @@ class ConsistencyClassifier(QWidget):
             self._all_scores = self._dataworker.scores
             self._frames = self._dataworker.frames
             self._tracks = self._dataworker.tracks
+            self._dataworker = None
+        if np.sum(self._userlabeled) < 1:
+            logging.error("ConsistencyTracker: I need at least 1 user-labeled frame to start with!")
+            self.setEnabled(False)
+        else:
+            t1_userlabeled = self._frames[self._userlabeled & (self._tracks == 1)]
+            t2_userlabeled = self._frames[self._userlabeled & (self._tracks == 2)]
+            max_startframe = np.min([t1_userlabeled[-1], t2_userlabeled[-1]])
+            min_startframe = np.max([t1_userlabeled[0], t2_userlabeled[0]])
             self._maxframes = np.max(self._frames)
-            # FIXME the following line causes an error when there are no detections in the range
-            min_frame = max([self._frames[self._tracks == 1][0], self._frames[self._tracks == 2][0]]) + 1
             self._maxframeslabel.setText(str(self._maxframes))
-            self._startframe_spinner.setMinimum(min_frame)
+            self._startframe_spinner.setMinimum(min_startframe)
-            self._startframe_spinner.setMaximum(self._frames[-1])
+            self._startframe_spinner.setMaximum(max_startframe)
-            self._startframe_spinner.setValue(self._frames[0] + 1)
+            self._startframe_spinner.setValue(min_startframe)
+            self._startframe_spinner.setSingleStep(20)
             self._startbtn.setEnabled(True)
             self._assignedlabel.setText("0")
             self._errorlabel.setText("0")
-            self._dataworker = None
+            self.setEnabled(True)
-        self.setEnabled(True)
 
     @Slot(float)
     def on_progress(self, value):
@@ -607,14 +666,14 @@ def main():
     import pickle
     from fixtracks.info import PACKAGE_ROOT
 
-    datafile = PACKAGE_ROOT / "data/merged2.pkl"
+    datafile = PACKAGE_ROOT / "data/merged_small_beginning.pkl"
 
     with open(datafile, "rb") as f:
         df = pickle.load(f)
-    data = TrackingData()
+    data = TrackingData(as_dict(df))
-    data.setData(as_dict(df))
     coords = data.coordinates()
-
+    cogs = data.centerOfGravity()
+    userlabeled = data["userlabeled"]
     app = QApplication([])
     window = QWidget()
     window.setMinimumSize(200, 200)
@@ -624,7 +683,7 @@ def main():
     # else:
     w = ClassifierWidget()
     w.setData(data)
-    w.size_classifier.setCoordinates(coords)
+    # w.size_classifier.setCoordinates(coords)
 
     layout = QVBoxLayout()
     layout.addWidget(w)

fixtracks/widgets/detectiontimeline.py

@@ -30,7 +30,7 @@ class Window(QGraphicsRectItem):
     def setWindowX(self, newx):
         logging.debug("timeline.window: set position to %.3f", newx)
         self.setX(newx)
-        self.signals.windowMoved.emit()
+        # self.signals.windowMoved.emit()
 
     def setWindowWidth(self, newwidth):
         logging.debug("timeline.window: update window width to %f", newwidth)
@@ -38,7 +38,7 @@ class Window(QGraphicsRectItem):
         r = self.rect()
         r.setWidth(newwidth)
         self.setRect(r)
-        self.signals.windowMoved.emit()
+        # self.signals.windowMoved.emit()
 
     def setWindow(self, newx:float, newwidth:float):
         """
@@ -58,7 +58,7 @@ class Window(QGraphicsRectItem):
         r = self.rect()
         self.setRect(newx, r.y(), self._width, r.height())
         self.update()
-        self.signals.windowMoved.emit()
+        # self.signals.windowMoved.emit()
 
     def mouseMoveEvent(self, event):
         super().mouseMoveEvent(event)
@@ -77,7 +77,7 @@ class Window(QGraphicsRectItem):
         if r.y() != self._y:
             self.setY(self._y)
         super().mouseReleaseEvent(event)
-        self.signals.windowMoved.emit()
+        self.signals.manualMove.emit()
 
     def hoverEnterEvent(self, event):
         super().hoverEnterEvent(event)
@@ -116,7 +116,7 @@ class DetectionTimeline(QWidget):
         font.setBold(True)
 
         self._window = Window(0, 0, 100, 60, window_pen, transparent_brush)
-        self._window.signals.windowMoved.connect(self.on_windowMoved)
+        self._window.signals.manualMove.connect(self.on_windowMoved)
 
         self._scene = QGraphicsScene(QRectF(0, 0, self._total_width, 85.))
         self._scene.setBackgroundBrush(self._bg_brush)

fixtracks/widgets/tracks.py

@@ -31,6 +31,7 @@ class FixTracks(QWidget):
         self._currentWindowPos = 0  # in frames
         self._currentWindowWidth = 0  # in frames
         self._maxframes = 0
+        self._manualmove = False
         self._data = None
 
         self._detectionView = DetectionView()
@@ -161,9 +162,10 @@ class FixTracks(QWidget):
     def update(self):
         start_frame = self._currentWindowPos
         stop_frame = start_frame + self._currentWindowWidth
+        self._timeline.setWindow(start_frame / self._maxframes,
+                                 self._currentWindowWidth/self._maxframes)
         logging.debug("Tracks:update: Updating View for detection range %i, %i frames", start_frame, stop_frame)
         self._data.setSelectionRange("frame", start_frame, stop_frame)
-
         self._controls_widget.setWindow(start_frame, stop_frame)
         kp = self._keypointcombo.currentText().lower()
         kpi = -1 if "center" in kp else int(kp)
@@ -208,11 +210,11 @@ class FixTracks(QWidget):
             self._saveBtn.setEnabled(True)
             self._currentWindowPos = 0
             self._currentWindowWidth = self._windowspinner.value()
-            self._maxframes = self._data.max("frame")
+            self._maxframes = np.max(self._data["frame"])
             self.populateKeypointCombo(self._data.numKeypoints())
             self._timeline.setData(self._data)
-            self._timeline.setWindow(self._currentWindowPos / self._maxframes,
+            # self._timeline.setWindow(self._currentWindowPos / self._maxframes,
-                                     self._currentWindowWidth / self._maxframes)
+            #                          self._currentWindowWidth / self._maxframes)
             self._detectionView.setData(self._data)
             self._classifier.setData(self._data)
             self.update()
@@ -276,7 +278,7 @@ class FixTracks(QWidget):
 
     def on_revertUserFlags(self):
         logging.debug("Tracks:revert ALL UserFlags and track assignments")
-        self._data.revertAssignmentStatus()
+        self._data.revertUserLabeledStatus()
         self._data.revertTrackAssignments()
         self._timeline.update()
         self.update()
@@ -289,8 +291,10 @@ class FixTracks(QWidget):
 
     def on_windowChanged(self):
         logging.debug("Tracks:Timeline reports window change ")
-        self._currentWindowPos = np.round(self._timeline.rangeStart * self._maxframes)
+        if not self._manualmove:
-        self.update()
+            self._currentWindowPos = np.round(self._timeline.rangeStart * self._maxframes)
+            self.update()
+        self._manualmove = False
 
     def on_windowSizeChanged(self, value):
         """Reacts on the user window-width selection. Selection is done in the unit of frames.
@@ -306,8 +310,7 @@ class FixTracks(QWidget):
         self._controls_widget.setSelectedTracks(None)
 
     def on_detectionsSelected(self, detections):
-        logging.debug("Tracks: Detections selected")
+        logging.debug("Tracks: %i Detections selected", len(detections))
-        print(detections)
         tracks = np.zeros(len(detections), dtype=int)
         ids = np.zeros_like(tracks)
         frames = np.zeros_like(tracks)
@@ -325,12 +328,11 @@ class FixTracks(QWidget):
         self._controls_widget.setSelectedTracks(tracks)
         self._skeleton.clear()
         self._skeleton.addSkeletons(coordinates, ids, frames, tracks, QBrush(QColor(10, 255, 65, 255)))
-        self.update()
 
     def moveWindow(self, stepsize):
-        step = np.round(stepsize * (self._currentWindowWidth))
+        logging.info("Tracks.moveWindow: move window with stepsize %.2f", stepsize)
-        new_start_frame = self._currentWindowPos + step
+        self._manualmove = True
-        self._timeline.setWindowPos(new_start_frame / self._maxframes)
+        new_start_frame = self._currentWindowPos + np.round(stepsize * self._currentWindowWidth)
         self._currentWindowPos = new_start_frame
         self._controls_widget.setSelectedTracks(None)
         self.update()