foreign fish detection using the chirp response seems to work

chirp_ams.py

@@ -25,7 +25,9 @@ def get_signals(eodfs, condition, contrast, chirp_size, chirp_duration, chirp_am
     _, chirper_signal, _, chirper_freq_profile = create_chirp(eodf=chirper_freq,
                                                               chirpsize=chirp_size, 
                                                               chirpduration=chirp_duration,
-                                                              ampl_reduction=chirp_amplitude_dip, chirptimes=chirp_times, duration=duration, dt=dt)
+                                                              ampl_reduction=chirp_amplitude_dip, 
+                                                              chirptimes=chirp_times,
+                                                              duration=duration, dt=dt)
     
     other_ampl = contrast/100
     if condition == "self":

response_discriminability.py

@@ -1,5 +1,6 @@
 import os
 import glob
+import pandas as pd
 import nixio as nix
 import numpy as np
 import scipy.signal as sig 
@@ -223,8 +224,18 @@ def create_response_plot(block_map, all_dfs, all_contrasts, all_conditions, curr
     plt.close()
 
 
-def foreign_fish_detection_beat(block_map, df, all_contrasts, all_conditions, kernel_width=0.0005):
+def get_chirp_metadata(block):
-    detection_performance = {} 
+    trial_duration = float(block.metadata["stimulus parameter"]["duration"])
+    dt = float(block.metadata["stimulus parameter"]["dt"])
+    chirp_duration = block.metadata["stimulus parameter"]["chirp_duration"]
+    chirp_size = block.metadata["stimulus parameter"]["chirp_size"]
+    chirp_times = block.metadata["stimulus parameter"]["chirp_times"]
+        
+    return trial_duration, dt, chirp_size, chirp_duration, chirp_times
+
+
+def foreign_fish_detection_beat(block_map, df, all_contrasts, all_conditions, kernel_width=0.0005, cell_name=""):
+    detection_performances = [] 
 
     for contrast in all_contrasts:
         print(" " * 50, end="\r")
@@ -232,11 +243,8 @@ def foreign_fish_detection_beat(block_map, df, all_contrasts, all_conditions, ke
         no_other_block = block_map[(contrast, df, "no-other")]
         self_block = block_map[(contrast, df, "self")]
         
-        # get some metadata assuming they are all the same for each conditionm, which they should
+        # get some metadata assuming they are all the same for each condition, which they should
-        duration = float(self_block.metadata["stimulus parameter"]["duration"])
+        duration, dt, _, chirp_duration, chirp_times = get_chirp_metadata(self_block)
-        dt = float(self_block.metadata["stimulus parameter"]["dt"])
-        chirp_duration = self_block.metadata["stimulus parameter"]["chirp_duration"]
-        chirp_times = self_block.metadata["stimulus parameter"]["chirp_times"]
         
         interchirp_starts = np.add(chirp_times, 1.5 * chirp_duration)[:-1]
         interchirp_ends = np.subtract(chirp_times, 1.5 * chirp_duration)[1:]
@@ -277,25 +285,117 @@ def foreign_fish_detection_beat(block_map, df, all_contrasts, all_conditions, ke
         score = np.hstack((valid_distances_baseline, valid_distances_comparison))
         fpr, tpr, _ = roc_curve(group, score, pos_label=1)
         auc = roc_auc_score(group, score)
-        detection_performance[(contrast, kernel_width)] = {"auc": auc, "true positives": tpr, "false positives": fpr}
+        detection_performances.append({"cell": cell_name, "detection_task": "beat", "contrast": contrast, "df": df, "kernel_width": kernel_width, "auc": auc, "true positives": tpr, "false positives": fpr})
+    print("\n")
+    return detection_performances
+
+
+def foreign_fish_detection_chirp(block_map, df, all_contrasts, all_conditions, kernel_width=0.0005, cell_name=""):
+    detection_performances = []
+
+    for contrast in all_contrasts:
+        print(" " * 50, end="\r")
+        print("Contrast: %.3f" % contrast, end="\r")
+        no_other_block = block_map[(contrast, df, "no-other")]
+        self_block = block_map[(contrast, df, "self")]
+        other_block = block_map[(contrast, df, "self")]
+        
+        # get some metadata assuming they are all the same for each condition, which they should
+        duration, dt, _, chirp_duration, chirp_times = get_chirp_metadata(self_block)
+                
+        # get the spiking responses
+        no_other_spikes = get_spikes(no_other_block)
+        self_spikes = get_spikes(self_block)
+        other_spikes = get_spikes(other_block)
+        
+        # get firing rates
+        no_other_rates, _ = get_rates(no_other_spikes, duration, dt, kernel_width)
+        self_rates, _ = get_rates(self_spikes, duration, dt, kernel_width)
+        other_rates, _ = get_rates(other_spikes, duration, dt, kernel_width)
+
+        # get the chirp response snippets
+        alone_chirping_snippets = np.zeros((len(chirp_times) * no_other_rates.shape[0], int(chirp_duration / dt)))
+        self_snippets = np.zeros_like(alone_chirping_snippets)
+        other_snippets = np.zeros_like(alone_chirping_snippets)
+        baseline_snippets = np.zeros_like(alone_chirping_snippets)
+
+        for i in range(no_other_rates.shape[0]):
+            for j, chirp_time in enumerate(chirp_times):
+                start_index = int((chirp_time - chirp_duration/2 + 0.003)/dt)
+                end_index = start_index + alone_chirping_snippets.shape[1]
+                index = i * len(chirp_times) + j
+                alone_chirping_snippets[index, :] = no_other_rates[i, start_index:end_index]
+                self_snippets[index, :] = self_rates[i, start_index:end_index]
+                other_snippets[index, :] = other_rates[i, start_index:end_index]
+                baseline_start_index = int((chirp_time + 1.5 * chirp_duration)/dt)
+                baseline_end_index = baseline_start_index + alone_chirping_snippets.shape[1]
+                baseline_snippets[index, :] = no_other_rates[i, baseline_start_index:baseline_end_index]
+        
+        # get the distances 
+        # 1. Soliloquy
+        # 2. Nobody chirps, all alone aka baseline response
+        # 3. I chirp while the other is present compared to self chirping without the other one present
+        # 4. the otherone chrips to me compared to baseline with anyone chirping
+        alone_chirping_dist = within_group_distance(alone_chirping_snippets)  
+        baseline_dist = within_group_distance(baseline_snippets)
+        self_vs_alone_dist = across_group_distance(alone_chirping_snippets, self_snippets)
+        other_vs_baseline_dist = across_group_distance(baseline_snippets, other_snippets)
+
+        # sort and perfom roc for two comparisons
+        # 1. soliloquy vs. self chirping in company
+        # 2. other chirping vs. nobody is chirping
+        triangle_indices = np.tril_indices_from(alone_chirping_dist, -1)
+        valid_no_other_distances = alone_chirping_dist[triangle_indices]
+        no_other_temp = np.zeros_like(valid_no_other_distances)
+
+        valid_baseline_distances = baseline_dist[triangle_indices]
+        baseline_temp = np.zeros_like(valid_baseline_distances)
+
+        valid_self_vs_alone_distances = self_vs_alone_dist.ravel()
+        self_vs_alone_temp = np.ones_like(valid_self_vs_alone_distances)
+
+        valid_other_vs_baseline_distances = other_vs_baseline_dist.ravel()
+        other_vs_baseline_temp = np.ones_like(valid_other_vs_baseline_distances)
+
+        group = np.hstack((no_other_temp, self_vs_alone_temp))
+        score = np.hstack((valid_no_other_distances, valid_self_vs_alone_distances))
+        fpr, tpr, _ = roc_curve(group, score, pos_label=1)
+        auc = roc_auc_score(group, score)
+        detection_performances.append({"cell": cell_name, "detection_task": "self vs soliloquy", "contrast": contrast, "df": df, "kernel_width": kernel_width, "auc": auc, "true positives": tpr, "false positives": fpr})
+        
+        group = np.hstack((baseline_temp, other_vs_baseline_temp))
+        score = np.hstack((valid_baseline_distances, valid_other_vs_baseline_distances))
+        fpr, tpr, _ = roc_curve(group, score, pos_label=1)
+        auc = roc_auc_score(group, score)
+        detection_performances.append({"cell": cell_name, "detection_task": "other vs quietness", "contrast": contrast, "df": df, "kernel_width": kernel_width, "auc": auc, "true positives": tpr, "false positives": fpr})
+
     print("\n")
-    return detection_performance
+    return detection_performances
 
 
-def foreign_fish_detection_chirp(block_map, df, all_contrasts, all_conditions, kernel_width=0.0005):
+def plot_detection_results(detection_beat, detection_chirp):
-    # 
-    return None
 
+    pass
 
-def foreign_fish_detection(block_map, all_dfs, all_contrasts, all_conditions, current_df=None, kernel_width=0.0005):
+
+def foreign_fish_detection(block_map, all_dfs, all_contrasts, all_conditions, current_df=None, kernel_width=0.0005, cell_name=""):
     dfs = [current_df] if current_df is not None else all_dfs 
+    result_dicts = []
     detection_performance_beat = {}
     detection_performance_chirp = {}
     for df in dfs:
-        detection_performance_beat[df] = foreign_fish_detection_beat(block_map, df, all_contrasts, all_conditions, kernel_width)
+        print("df: %i" % df)
-        detection_performance_chirp[df] = foreign_fish_detection_chirp(block_map, df, all_contrasts, all_conditions, kernel_width)
+        print("Foreign fish detection during beat:")
+        result_dicts.extend(foreign_fish_detection_beat(block_map, df, all_contrasts, all_conditions, kernel_width, cell_name))
+        print("Foreign fish detection during chirp:")
+        result_dicts.extend(foreign_fish_detection_chirp(block_map, df, all_contrasts, all_conditions, kernel_width, cell_name))
+        
+
+        break
+
+    embed()
  
-    return detection_performance_beat, detection_performance_chirp
+    return result_dicts
 
 
 def process_cell(filename, dfs=[], contrasts=[], conditions=[]):
@@ -309,7 +409,7 @@ def process_cell(filename, dfs=[], contrasts=[], conditions=[]):
     # create_response_plot(block_map, all_dfs, all_contrasts, all_conditions, 20, figure_name=fig_name)
     # fig_name = filename.split(os.path.sep)[-1].split(".nix")[0] + "_df_-100Hz.pdf"
     # create_response_plot(block_map, all_dfs, all_contrasts, all_conditions, -100, figure_name=fig_name)
-    foreign_fish_detection(block_map, all_dfs, all_contrasts, all_conditions, current_df=20)
+    results = foreign_fish_detection(block_map, all_dfs, all_contrasts, all_conditions, current_df=20, cell_name=filename.split(os.path.sep)[-1].split(".nix")[0])
 
     
     nf.close()