csachgau/fishBook
1
0
Fork 0
forked from jgrewe/fishbook
Code Issues Pull Requests Releases Wiki Activity

[backend] a few improvements, user feedback during import

Browse Source
This commit is contained in:
Jan Grewe 2020-07-31 18:05:24 +02:00
parent b43aabba6f
commit 9d61ca7a1c
1 changed files with 33 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
fishbook/backend/database.py
View File

@ -6,10 +6,13 @@ import glob
import socket import socket
from fishbook.backend.util import read_info_file, read_dataset_info, read_stimuli_file from fishbook.backend.util import read_info_file, read_dataset_info, read_stimuli_file
from fishbook.backend.util import find_key_recursive, deep_get, find_mtags_for_tag from fishbook.backend.util import find_key_recursive, deep_get, find_mtags_for_tag
from fishbook.backend.util import mtag_settings_to_yaml, nix_metadata_to_yaml, progress from fishbook.backend.util import mtag_settings_to_yaml, nix_metadata_to_yaml, mtag_features_to_yaml, progress
import uuid import uuid
import yaml import yaml
from IPython import embed
dj.config["enable_python_native_blobs"] = True
schema = dj.schema("fish_book", locals()) schema = dj.schema("fish_book", locals())
@ -129,6 +132,7 @@ class SubjectProperties(dj.Manual):
eod_frequency : float eod_frequency : float
""" """
@staticmethod
def get_template_tuple(id=None): def get_template_tuple(id=None):
tup = dict(id=None, subject_id=None, recording_date=None, weight=0.0, size=0.0, tup = dict(id=None, subject_id=None, recording_date=None, weight=0.0, size=0.0,
eod_frequency=0.0) eod_frequency=0.0)
@ -316,8 +320,11 @@ def populate_cells(data_path):
cell_info = deep_get(info, p) cell_info = deep_get(info, p)
p = [] p = []
find_key_recursive(info, "Firing Rate1", p) res = find_key_recursive(info, "Firing Rate1", p)
if res:
firing_rate = deep_get(info, p, default=0.0) firing_rate = deep_get(info, p, default=0.0)
else:
firing_rate = 0.0
if isinstance(firing_rate, str): if isinstance(firing_rate, str):
firing_rate = float(firing_rate[:-2]) firing_rate = float(firing_rate[:-2])
@ -356,7 +363,6 @@ def populate_cells(data_path):
cell_props["lateral_pos"] = float(cell_info["Lateral position"][:-2]) cell_props["lateral_pos"] = float(cell_info["Lateral position"][:-2])
if "Transverse section" in cell_info.keys(): if "Transverse section" in cell_info.keys():
cell_props["transversal_section"] = float(cell_info["Transverse section"]) cell_props["transversal_section"] = float(cell_info["Transverse section"])
Cells.insert1(cell_props, skip_duplicates=True) Cells.insert1(cell_props, skip_duplicates=True)
# multi match entry # multi match entry
@ -377,11 +383,12 @@ def scan_nix_file_for_repros(dataset):
repro_runs = [t for t in b.tags if "relacs.repro_run" in t.type] repro_runs = [t for t in b.tags if "relacs.repro_run" in t.type]
total = len(repro_runs) total = len(repro_runs)
for i, t in enumerate(repro_runs): for i, t in enumerate(repro_runs):
progress(i+1, total, "Scanning repro run %s" % rs["RePro"])
rs = t.metadata.find_sections(lambda x: "Run" in x.props) rs = t.metadata.find_sections(lambda x: "Run" in x.props)
rs = rs[0]
if len(rs) == 0: if len(rs) == 0:
continue continue
rs = rs[0]
progress(i+1, total, "Scanning repro run %s" % rs["RePro"])
rp = Repros.get_template_tuple() rp = Repros.get_template_tuple()
rp["run"] = rs["Run"] rp["run"] = rs["Run"]
@ -405,11 +412,24 @@ def scan_nix_file_for_repros(dataset):
repro.pop("duration") repro.pop("duration")
mtags, positions = find_mtags_for_tag(b, t) mtags, positions = find_mtags_for_tag(b, t)
for i, mt in enumerate(mtags): mt_settings_dict = {}
mt_positions = np.atleast_2d(mt.positions[:]).T positions_dict = {}
mt_extents = np.atleast_2d(mt.extents[:]).T extents_dict = {}
for p in positions[i]: for j, mt in enumerate(mtags):
settings = mtag_settings_to_yaml(mt, p) if mt.id in positions_dict.keys():
mt_positions = positions_dict[mt.id]
mt_extents = extents_dict[mt.id]
mdata_yaml = mt_settings_dict[mt.id]
else:
mdata_yaml = nix_metadata_to_yaml(mt.metadata)
mt_settings_dict[mt.id] = mdata_yaml
mt_positions = np.atleast_2d(mt.positions[:])
mt_extents = np.atleast_2d(mt.extents[:])
if mt.positions.shape[0] != mt_positions.shape[0]:
mt_positions = mt_positions.T
mt_extents = mt_extents.T
for p in positions[j]:
settings = mtag_features_to_yaml(mt, p, mdata_yaml)
stim_start = mt_positions[p, 0] stim_start = mt_positions[p, 0]
stim_duration = mt_extents[p, 0] stim_duration = mt_extents[p, 0]
@ -424,6 +444,7 @@ def scan_nix_file_for_repros(dataset):
stim["stimulus_name"] = mt.name stim["stimulus_name"] = mt.name
stim.update(repro) stim.update(repro)
Stimuli.insert1(stim, skip_duplicates=True) Stimuli.insert1(stim, skip_duplicates=True)
print(" " * 120, end="\r")
print("\n") print("\n")
f.close() f.close()
f = None f = None
@ -434,7 +455,7 @@ def scan_folder_for_repros(dataset):
repro_settings, stim_indices = read_stimuli_file(dataset["data_source"]) repro_settings, stim_indices = read_stimuli_file(dataset["data_source"])
repro_counts = {} repro_counts = {}
cell_id = (Cells * CellDatasetMap * (Datasets & "dataset_id = '%s'" % dataset["dataset_id"])).fetch("cell_id", limit=1)[0] cell_id = (Cells * CellDatasetMap * (Datasets & "dataset_id = '%s'" % dataset["dataset_id"])).fetch("cell_id", limit=1)[0]
for i, (rs, si) in enumerate(zip(repro_settings, stim_indices)): for rs, si in zip(repro_settings, stim_indices):
rp = Repros.get_template_tuple() rp = Repros.get_template_tuple()
path = [] path = []
if not find_key_recursive(rs, "run", path): if not find_key_recursive(rs, "run", path):