diff --git a/fishbook/backend/database.py b/fishbook/backend/database.py index dbc989f..ce847bf 100644 --- a/fishbook/backend/database.py +++ b/fishbook/backend/database.py @@ -6,10 +6,13 @@ import glob import socket from fishbook.backend.util import read_info_file, read_dataset_info, read_stimuli_file from fishbook.backend.util import find_key_recursive, deep_get, find_mtags_for_tag -from fishbook.backend.util import mtag_settings_to_yaml, nix_metadata_to_yaml, progress +from fishbook.backend.util import mtag_settings_to_yaml, nix_metadata_to_yaml, mtag_features_to_yaml, progress import uuid import yaml +from IPython import embed + +dj.config["enable_python_native_blobs"] = True schema = dj.schema("fish_book", locals()) @@ -129,6 +132,7 @@ class SubjectProperties(dj.Manual): eod_frequency : float """ + @staticmethod def get_template_tuple(id=None): tup = dict(id=None, subject_id=None, recording_date=None, weight=0.0, size=0.0, eod_frequency=0.0) @@ -316,8 +320,11 @@ def populate_cells(data_path): cell_info = deep_get(info, p) p = [] - find_key_recursive(info, "Firing Rate1", p) - firing_rate = deep_get(info, p, default=0.0) + res = find_key_recursive(info, "Firing Rate1", p) + if res: + firing_rate = deep_get(info, p, default=0.0) + else: + firing_rate = 0.0 if isinstance(firing_rate, str): firing_rate = float(firing_rate[:-2]) @@ -356,7 +363,6 @@ def populate_cells(data_path): cell_props["lateral_pos"] = float(cell_info["Lateral position"][:-2]) if "Transverse section" in cell_info.keys(): cell_props["transversal_section"] = float(cell_info["Transverse section"]) - Cells.insert1(cell_props, skip_duplicates=True) # multi match entry @@ -377,12 +383,13 @@ def scan_nix_file_for_repros(dataset): repro_runs = [t for t in b.tags if "relacs.repro_run" in t.type] total = len(repro_runs) for i, t in enumerate(repro_runs): - progress(i+1, total, "Scanning repro run %s" % rs["RePro"]) rs = t.metadata.find_sections(lambda x: "Run" in x.props) + rs = rs[0] if len(rs) == 0: continue - rs = rs[0] + progress(i+1, total, "Scanning repro run %s" % rs["RePro"]) + rp = Repros.get_template_tuple() rp["run"] = rs["Run"] rp["repro_name"] = rs["RePro"] @@ -405,11 +412,24 @@ def scan_nix_file_for_repros(dataset): repro.pop("duration") mtags, positions = find_mtags_for_tag(b, t) - for i, mt in enumerate(mtags): - mt_positions = np.atleast_2d(mt.positions[:]).T - mt_extents = np.atleast_2d(mt.extents[:]).T - for p in positions[i]: - settings = mtag_settings_to_yaml(mt, p) + mt_settings_dict = {} + positions_dict = {} + extents_dict = {} + for j, mt in enumerate(mtags): + if mt.id in positions_dict.keys(): + mt_positions = positions_dict[mt.id] + mt_extents = extents_dict[mt.id] + mdata_yaml = mt_settings_dict[mt.id] + else: + mdata_yaml = nix_metadata_to_yaml(mt.metadata) + mt_settings_dict[mt.id] = mdata_yaml + mt_positions = np.atleast_2d(mt.positions[:]) + mt_extents = np.atleast_2d(mt.extents[:]) + if mt.positions.shape[0] != mt_positions.shape[0]: + mt_positions = mt_positions.T + mt_extents = mt_extents.T + for p in positions[j]: + settings = mtag_features_to_yaml(mt, p, mdata_yaml) stim_start = mt_positions[p, 0] stim_duration = mt_extents[p, 0] @@ -424,6 +444,7 @@ def scan_nix_file_for_repros(dataset): stim["stimulus_name"] = mt.name stim.update(repro) Stimuli.insert1(stim, skip_duplicates=True) + print(" " * 120, end="\r") print("\n") f.close() f = None @@ -434,7 +455,7 @@ def scan_folder_for_repros(dataset): repro_settings, stim_indices = read_stimuli_file(dataset["data_source"]) repro_counts = {} cell_id = (Cells * CellDatasetMap * (Datasets & "dataset_id = '%s'" % dataset["dataset_id"])).fetch("cell_id", limit=1)[0] - for i, (rs, si) in enumerate(zip(repro_settings, stim_indices)): + for rs, si in zip(repro_settings, stim_indices): rp = Repros.get_template_tuple() path = [] if not find_key_recursive(rs, "run", path):