diff --git a/database.py b/database.py index ebafd93..d3b8546 100644 --- a/database.py +++ b/database.py @@ -19,6 +19,8 @@ class Dataset(dj.Manual): data_source : varchar(512) # path to the dataset experimenter : varchar(512) recording_date : date + quality : varchar(512) + comment : varchar(1024) has_nix : bool """ @@ -27,7 +29,8 @@ class Dataset(dj.Manual): if id is not None: d = dict((Dataset() & {"dataset_id": id}).fetch1()) return d - return dict(dataset_id=None, data_source="", experimenter="", recording_date=None, has_nix=False) + return dict(dataset_id=None, data_source="", experimenter="", recording_date=None, + quality="", comment="", has_nix=False) @staticmethod def get_nix_file(key): @@ -124,9 +127,43 @@ class SubjectProperties(dj.Manual): return tup +@schema +class Cell(dj.Manual): + definition = """ + # Table that stores information about recorded cells. + id : int auto_increment + ---- + -> Subject + -> Dataset + cell_name : varchar(256) + cell_type : varchar(256) + firing_rate : float + structure : varchar(256) + region : varchar(256) + subregion : varchar(256) + depth : float + lateral_pos : float + transversal_section : float + """ + + @staticmethod + def get_template_tuple(cell_id=None): + tup = dict(id=None, dataset_id=None, subject_id=None, cell_name="", cell_type="", firing_rate=0.0, depth=0.0, + region="", subregion="", structure="", lateral_pos=0.0, transversal_section=0.0) + if cell_id is not None: + d = dict((Cell() & {"cell_id": cell_id}).fetch1()) + return d + return tup + + def read_info(info_file): + exp = "" + quality = "" + comment = "" + rec_date = None + has_nix = False if not os.path.exists(info_file): - return None, None, False + return exp, rec_date, quality, comment, has_nix has_nix = len(glob.glob(os.path.sep.join(info_file.split(os.path.sep)[:-1]) + os.path.sep + "*.nix")) > 0 info = read_info_file(info_file) p = [] @@ -137,7 +174,15 @@ def read_info(info_file): find_key_recursive(info, "Date", p) if len(p) > 0: rec_date = dt.date.fromisoformat(deep_get(info, p)) - return exp, rec_date, has_nix + p = [] + find_key_recursive(info, "Recording quality", p) + if len(p) > 0: + quality = deep_get(info, p) + find_key_recursive(info, "Comment", p) + if len(p) > 0: + comment = deep_get(info, p, default="") + + return exp, rec_date, quality, comment, has_nix def populate_datasets(data_path): @@ -145,7 +190,7 @@ def populate_datasets(data_path): if not os.path.exists(data_path): return dset_name = os.path.split(data_path)[-1] - experimenter, rec_date, has_nix = read_info(os.path.join(data_path, 'info.dat')) + experimenter, rec_date, quality, comment, has_nix = read_info(os.path.join(data_path, 'info.dat')) if not experimenter: return @@ -154,8 +199,9 @@ def populate_datasets(data_path): inserts["data_source"] = data_path inserts["experimenter"] = experimenter inserts["recording_date"] = rec_date + inserts["quality"] = quality + inserts["comment"] = comment inserts["has_nix"] = has_nix - Dataset().insert1(inserts, skip_duplicates=True) @@ -196,12 +242,65 @@ def populate_subjects(data_path): SubjectProperties.insert1(props, skip_duplicates=True) -if __name__ == "__main__": - datasets = glob.glob('/data/apteronotus/2018-05-08*') - # Dataset.drop() - # Subject.drop() - # SubjectProperties.drop() - # SubjectDatasetMap.drop() +def populate_cells(data_path): + print("Importing subject(s) of %s" % data_path) + dset_name = os.path.split(data_path)[-1] + info_file = os.path.join(data_path, 'info.dat') + if not os.path.exists(info_file): + return None, None, False + info = read_info_file(info_file) + p = [] + find_key_recursive(info, "Subject", p) + subject_info = deep_get(info, p) + + p = [] + find_key_recursive(info, "Cell", p) + cell_info = deep_get(info, p) + + p = [] + find_key_recursive(info, "Firing Rate1", p) + firing_rate = deep_get(info, p, default=0.0) + if isinstance(firing_rate, str): + firing_rate = float(firing_rate[:-2]) + + dataset = dict((Dataset & {"dataset_id": dset_name}).fetch1()) + subject = dict((Subject & {"subject_id": subject_info["Identifier"]}).fetch1()) + + cell_props = Cell.get_template_tuple() + cell_props["subject_id"] = subject["subject_id"] + cell_props["dataset_id"] = dataset["dataset_id"] + cell_props["cell_name"] = dataset["dataset_id"] + cell_props["cell_type"] = cell_info["CellType"] + cell_props["firing_rate"] = firing_rate + if "Structure" in cell_info.keys(): + cell_props["structure"] = cell_info["Structure"] + if "BrainRegion" in cell_info.keys(): + cell_props["region"] = cell_info["BrainRegion"] + if "BrainSubRegion" in cell_info.keys(): + cell_props["subregion"] = cell_info["BrainSubRegion"] + if "Depth" in cell_info.keys(): + cell_props["depth"] = float(cell_info["Depth"][:-2]) + if "Lateral position" in cell_info.keys(): + cell_props["lateral_pos"] = float(cell_info["Lateral position"][:-2]) + if "Transverse section" in cell_info.keys(): + cell_props["transversal_section"] = float(cell_info["Transverse section"]) + + Cell.insert1(cell_props, skip_duplicates=True) + + +def drop_tables(): + Dataset.drop() + Subject.drop() + + +def populate(datasets): for d in datasets: populate_datasets(d) populate_subjects(d) + populate_cells(d) + + +if __name__ == "__main__": + datasets = glob.glob('/data/apteronotus/2018-05-08*') + # drop_tables() + populate(datasets)