diff --git a/baseline_analysis.py b/baseline_analysis.py new file mode 100644 index 0000000..7d04d3d --- /dev/null +++ b/baseline_analysis.py @@ -0,0 +1,16 @@ +import datajoint as dj +from database import Dataset, Repro +schema = dj.schema("fish_book", locals()) + + +def get_baseline_data(cell_id=None, restriction={}): + dsets = [] + if dataset_name: + dsets = (Dataset & ("dataset_id like'%s'" % dataset_name) & restriction) + else: + dsets = (Dataset & restriction) + + if len(dsets) == 0: + return None + for d in dsets: + d diff --git a/database.py b/database.py index 042afd8..c36cc87 100644 --- a/database.py +++ b/database.py @@ -172,7 +172,7 @@ class Repro(dj.Manual): definition = """ repro_id : varchar(512) run : smallint - -> Dataset + -> Dataset # this is wrong! should be cell!?! In our case it is the same... ---- repro_name : varchar(512) settings : varchar(3000) @@ -215,7 +215,6 @@ class Stimulus(dj.Manual): def populate_datasets(data_path, update=False): - print("Importing dataset %s" % data_path) if not os.path.exists(data_path): return dset_name = os.path.split(data_path)[-1] @@ -312,13 +311,21 @@ def populate_cells(data_path): if isinstance(firing_rate, str): firing_rate = float(firing_rate[:-2]) - if "Identifier" not in subject_info.keys() and ("Identifier" in info.keys() and - isinstance(info["Identifier"], dict)): - subject_id = "unspecified_" + dset_name + subj_id = None + if "Identifier" in subject_info.keys(): + if isinstance(subject_info["Identifier"], dict): + subj_id = "unspecified_" + dset_name + else: + subj_id = subject_info["Identifier"] + elif "Identifier" in info.keys(): + if isinstance(info["Identifier"], dict): + subj_id = "unspecified_" + dset_name + else: + subj_id = info["Identifier"] else: - subject_id = subject_info["Identifier"] + subj_id = "unspecified_" + dset_name dataset = dict((Dataset & {"dataset_id": dset_name}).fetch1()) - subject = dict((Subject & {"subject_id": subject_id}).fetch1()) + subject = dict((Subject & {"subject_id": subj_id}).fetch1()) dataset_id = dataset["dataset_id"] cell_id = "-".join(dataset_id.split("-")[:4]) if len(dataset_id) > 4 else dataset_id @@ -455,7 +462,10 @@ def scan_folder_for_repros(dataset): ut.find_key_recursive(rs, "Duration", path) if len(path) > 0 : stim_duration = ut.deep_get(rs, path, None) - stim_duration = float(stim_duration[:-2]) + if "ms" in stim_duration: + stim_duration = float(stim_duration[:stim_duration.index("ms")]) + else: + stim_duration = float(stim_duration[:stim_duration.index("s")]) else: stim_duration = 0.0 @@ -491,9 +501,10 @@ def drop_tables(): Subject.drop() -def populate(datasets): - for d in datasets: - if not populate_datasets(d): +def populate(datasets, update=False): + for i, d in enumerate(datasets): + print("Importing %i of %i: %s" % (i, len(datasets), d)) + if not populate_datasets(d, update): continue populate_subjects(d) populate_cells(d) @@ -508,6 +519,6 @@ if __name__ == "__main__": # data_dir = "../high_freq_chirps/data" # drop_tables() - datasets = glob.glob(os.path.join(data_dir, '/data/apteronotus/201*')) - populate(datasets) + datasets = glob.glob(os.path.join(data_dir, '/data/eigenmannia/201*')) + populate(datasets, update=False) diff --git a/util.py b/util.py index 0bee4ff..4920332 100644 --- a/util.py +++ b/util.py @@ -5,6 +5,7 @@ import re import os import glob import datetime as dt +import subprocess from IPython import embed @@ -15,20 +16,28 @@ def read_info_file(file_name): @return: dictionary, the stored information. """ root = {} - with open(file_name, 'r') as f: - lines = f.readlines() - for l in lines: - if not l.startswith("#"): - continue - l = l.strip("#").strip() - if len(l) == 0: - continue - if not ": " in l: # subsection - sec = {} - root[l[:-1] if l.endswith(":") else l] = sec - else: - parts = l.split(': ') - sec[parts[0].strip()] = parts[1].strip('"').strip() + + try: + with open(file_name, 'r') as f: + lines = f.readlines() + except UnicodeDecodeError: + print("Replacing experimenter!!!") + command = "sudo sed -i '/Experimenter/c\# Experimenter: Anna Stoeckl' %s" % file_name + subprocess.check_call(command, shell=True) + with open(file_name, 'r') as f: + lines = f.readlines() + for l in lines: + if not l.startswith("#"): + continue + l = l.strip("#").strip() + if len(l) == 0: + continue + if not ": " in l: # subsection + sec = {} + root[l[:-1] if l.endswith(":") else l] = sec + else: + parts = l.split(': ') + sec[parts[0].strip()] = parts[1].strip('"').strip() return root