improve analysis

AnalysisMasterScript.py

@@ -1,13 +1,29 @@
 
 import os
-import Figures_Baseline
-import Figures_Stimuli
-import Figures_Model
-import Figures_results
+
+import Figures_results as Fr
+import Figure_constants as Fc
+import my_util.save_load as sl
+
+
+fit_folder = "results/sam_cells"  # kraken fit
+# fit_folder = "results/final_sam_dend_noise_test/"  # noise in input fit
+# fit_folder = "results/final_sam2/"  # noise in input fit
+
+figure_folder = "temp/preliminary_figures/"
+pre_analysis_folder = "temp/pre_analysis/"
 
 
 def main():
-    pass
+
+    for folder in (figure_folder, pre_analysis_folder):
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+
+    Fc.SAVE_FOLDER = figure_folder
+
+    Fr.run_all_images(fit_folder, filter=False, pre_analysis_path=pre_analysis_folder, recalculate=True)
+
 
 if __name__ == '__main__':
     main()

Figures_results.py

@@ -2,14 +2,18 @@ import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib.gridspec as gridspec
 import matplotlib as mpl
+from scipy.stats import pearsonr
+import os
+
 from analysis import get_filtered_fit_info, get_behaviour_values, get_parameter_values, behaviour_correlations, parameter_correlations
 from fitting.ModelFit import get_best_fit
 from experiments.Baseline import BaselineModel, BaselineCellData
 from experiments.FiCurve import FICurveModel, FICurveCellData
 from parser.CellData import CellData
 from my_util import functions as fu
+from my_util import save_load
 import Figure_constants as consts
-from scipy.stats import pearsonr
+
 
 parameter_titles = {"input_scaling": r"$\alpha$", "delta_a": r"$\Delta_A$",
                     "mem_tau": r"$\tau_m$", "noise_strength": r"$\sqrt{2D}$",
@@ -58,12 +62,30 @@ def main():
     # example_bad_fi_fits(dir_path)
 
 
-def run_all_images():
-    dend_tau_and_ref_effect()
+def run_all_images(dir_path, filter=True,  pre_analysis_path="", recalculate=False):
 
-    dir_path = "results/final_2/"
-    fits_info = get_filtered_fit_info(dir_path, filter=True)
-    cell_behaviour, model_behaviour = get_behaviour_values(fits_info)
+
+    if pre_analysis_path != "":
+        fit_info_name = "figures_res_fit_info.npy"
+        behaviours_name = "figures_res_behaviour.npy"
+
+        fit_info_path = os.path.join(pre_analysis_path, fit_info_name)
+        if not os.path.exists(fit_info_path) or recalculate:
+            fits_info = get_filtered_fit_info(dir_path, filter=filter)
+            save_load.save(fits_info, fit_info_path)
+        else:
+            fits_info = save_load.load(fit_info_path)
+
+        behaviours_path = os.path.join(pre_analysis_path, behaviours_name)
+        if not os.path.exists(behaviours_path) or recalculate:
+            cell_behaviour, model_behaviour = get_behaviour_values(fits_info)
+            save_load.save([cell_behaviour, model_behaviour], behaviours_path)
+        else:
+            cell_behaviour, model_behaviour = save_load.load(behaviours_path)
+
+    else:
+        fits_info = get_filtered_fit_info(dir_path, filter=True)
+        cell_behaviour, model_behaviour = get_behaviour_values(fits_info)
 
     plot_cell_model_comp_baseline(cell_behaviour, model_behaviour)
     plot_cell_model_comp_adaption(cell_behaviour, model_behaviour)
@@ -75,10 +97,13 @@ def run_all_images():
     create_parameter_distributions(get_parameter_values(fits_info))
     create_parameter_distributions(get_parameter_values(fits_info, scaled=True, goal_eodf=800), "scaled_to_800_")
 
-    example_good_hist_fits(dir_path)
-    example_bad_hist_fits(dir_path)
-    example_good_fi_fits(dir_path)
-    example_bad_fi_fits(dir_path)
+    # Plots using example cells:
+
+    # dend_tau_and_ref_effect()
+    # example_good_hist_fits(dir_path)
+    # example_bad_hist_fits(dir_path)
+    # example_good_fi_fits(dir_path)
+    # example_bad_fi_fits(dir_path)
 
 
 def visualize_tested_correlations(fits_info):
@@ -629,6 +654,7 @@ def plot_cell_model_comp_baseline(cell_behavior, model_behaviour):
 
     ax = fig.add_subplot(gs[1, i])
     ax_histx = fig.add_subplot(gs[0, i], sharex=ax)
+    print("Cells in cell_model_comp_baseline:", len(cell))
     scatter_hist(cell, model, ax, ax_histx, behaviour_titles["vector_strength"], bins)  # , cmap, cell_bursting)
     ax.set_xlabel(r"Cell")
     ax.set_ylabel(r"Model")

analysis.py

@@ -114,6 +114,9 @@ def get_filtered_fit_info(folder, filter=True):
     for item in os.listdir(folder):
         cell_folder = os.path.join(folder, item)
 
+        if not os.path.isdir(cell_folder):
+            continue
+
         results = get_best_fit(cell_folder, use_comparable_error=False)
         cell_behaviour, model_behaviour = results.get_behaviour_values()
 

parser/DataParserFactory.py

@@ -79,6 +79,13 @@ class DatParser(AbstractParser):
     def has_sam_recordings(self):
         return exists(self.sam_file)
 
+    def get_measured_repros(self):
+        repros = []
+        for metadata, key, data in Dl.iload(self.stimuli_file):
+            repros.extend([d["repro"] for d in metadata if "repro" in d.keys()])
+
+        return sorted(np.unique(repros))
+
     def get_baseline_length(self):
         lengths = []
         for metadata, key, data in Dl.iload(self.baseline_file):

run_Fitter.py

@@ -5,13 +5,12 @@ from experiments.Baseline import get_baseline_class
 from experiments.FiCurve import get_fi_curve_class
 from fitting.Fitter import Fitter
 from fitting.ModelFit import get_best_fit, ModelFit
+from my_util.helperFunctions import plot_errors
 
 import time
 import os
 import argparse
 import numpy as np
-from my_util.helperFunctions import plot_errors
-
 import multiprocessing as mp
 
 SAVE_DIRECTORY = "./results/sam_cells/"

sam_experiments.py

@@ -10,7 +10,7 @@ import os
 
 
 def main():
-    run_sam_analysis_for_all_cells("results/final_sam2")
+    run_sam_analysis_for_all_cells("results/sam_cells")
 
     # sam_analysis("results/final_2/2011-10-25-ad-invivo-1/")
 

test.py

@@ -1,5 +1,6 @@
 import os
 from parser.CellData import CellData
+from parser.DataParserFactory import DatParser
 import numpy as np
 from fitting.ModelFit import ModelFit, get_best_fit
 # from plottools.axes import labelaxes_params
@@ -9,35 +10,43 @@ colors = ["black", "red", "blue", "orange", "green"]
 
 
 def main():
-    # sam_tests()
-    # cells = 40
-    number = len([i for i in iget_start_parameters()])
-    single_core = number * 1400 / 60 / 60
-    print("start parameters:", number)
-    print("single core time:", single_core, "h")
-    print("single core time:", single_core/24, "days")
-
-    cores = 16
-    cells = 40
 
-    print(cores, "core time:", single_core/cores, "h")
-    print(cores, "core time:", single_core / 24 / cores, "days")
-    print(cores, "core time all", cells, "cells:", single_core / 24 / cores * cells, "days")
+    fit = get_best_fit("results/kraken_fit/2011-10-25-ad-invivo-1/")
+    print(fit.get_fit_routine_error())
 
-    print("left over:", number%cores)
+    quit()
+    sam_tests()
+    # cells = 40
+    # number = len([i for i in iget_start_parameters()])
+    # single_core = number * 1400 / 60 / 60
+    # print("start parameters:", number)
+    # print("single core time:", single_core, "h")
+    # print("single core time:", single_core/24, "days")
+    #
+    # cores = 16
+    # cells = 40
+    #
+    # print(cores, "core time:", single_core/cores, "h")
+    # print(cores, "core time:", single_core / 24 / cores, "days")
+    # print(cores, "core time all", cells, "cells:", single_core / 24 / cores * cells, "days")
+    #
+    # print("left over:", number%cores)
 
     # fit = get_best_fit("results/final_sam2/2012-12-20-ae-invivo-1/")
     # fit.generate_master_plot()
 
 
 def sam_tests():
-    data_folder = "./data/final/"
+    data_folder = "./data/final_sam/"
     for cell in sorted(os.listdir(data_folder)):
         print(cell)
         cell_folder = os.path.join(data_folder, cell)
         if not os.path.exists(os.path.join(cell_folder, "samspikes1.dat")):
             continue
 
+        if "2018-05-08-aa-invivo-1" not in cell:
+            continue
+
         cell_data = CellData(cell_folder)
         sampling_rate = int(round(1 / cell_data.get_sampling_interval()))
         sam_spikes = cell_data.get_sam_spiketimes()
@@ -46,7 +55,8 @@ def sam_tests():
         [time_traces, v1_traces, eod_traces, local_eod_traces, stimulus_traces] = cell_data.get_sam_traces()
         print(len(time_traces))
         for i in range(len(delta_freqs)):
-
+            if abs(delta_freqs[i]) > 50:
+                continue
             fig, axes = plt.subplots(2, 1, sharex="all")
 
             axes[0].plot(time_traces[i], local_eod_traces[i])
@@ -59,7 +69,7 @@ def sam_tests():
                                   colors=colors[j % len(colors)])
             plt.show()
             plt.close()
-        break
+
 
 
 def average_spike_height(spike_trains, local_eod, sampling_rate):