csachgau/fishBook
1
0
Fork 0
forked from jgrewe/fishbook
Code Issues Pull Requests Releases Wiki Activity

kind of works for nix files

Browse Source
This commit is contained in:
Jan Grewe 2019-09-06 17:59:30 +02:00
parent dee40d879c
commit eba710b615
2 changed files with 95 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
database.py
View File

@ -5,6 +5,7 @@ import os
import glob
import util as ut
import uuid
import pyrelacs
from IPython import embed
@ -195,6 +196,7 @@ class Stimulus(dj.Manual):
---
stimulus_index : int
stimulus_name : varchar(512)
mtag_id : varchar(50)
start : float
duration : float
settings : varchar(3000)
@ -246,7 +248,7 @@ def populate_subjects(data_path):
inserts["species"] = subj["Species"]
Subject().insert1(inserts, skip_duplicates=True)
# multi mach entry
# multi match entry
dataset = dict((Dataset() & {"dataset_id": dset_name}).fetch1())
mm = dict(dataset_id=dataset["dataset_id"], subject_id=subj["Identifier"])
SubjectDatasetMap.insert1(mm, skip_duplicates=True)
@ -318,14 +320,7 @@ def populate_cells(data_path):
CellDatasetMap.insert1(mm, skip_duplicates=True)
def populate_repros(data_path):
print("\tImporting RePro(s) of %s" % data_path)
dset_name = os.path.split(data_path)[-1]
if len(Dataset & {"dataset_id": dset_name}) != 1:
return False
dataset = dict((Dataset & {"dataset_id": dset_name}).fetch1())
if dataset["has_nix"]:
def scan_nix_file_for_repros(dataset):
print("\t\tscanning nix file")
nix_files = glob.glob(os.path.join(dataset["data_source"], "*.nix"))
for nf in nix_files:
@ -334,8 +329,8 @@ def populate_repros(data_path):
continue
f = nix.File.open(nf, nix.FileMode.ReadOnly)
b = f.blocks[0]
for t in b.tags:
if "relacs.repro_run" in t.type:
repro_runs = [t for t in b.tags if "relacs.repro_run" in t.type]
for t in repro_runs:
rs = t.metadata.find_sections(lambda x: "Run" in x.props)
if len(rs) == 0:
continue
@ -353,35 +348,53 @@ def populate_repros(data_path):
rp["start"] = t.position[0]
rp["duration"] = t.extent[0]
Repro.insert1(rp, skip_duplicates=True)
repro = dict((Repro & dict(repro_id="BaselineActivity_2", dataset_id="2018-11-20-ae-invivo-1")).fetch1())
# import Stimuli
repro = dict((Repro & dict(repro_id=rp["repro_id"], dataset_id=rp["dataset_id"])).fetch1())
repro.pop("settings")
repro.pop("repro_name")
repro.pop("start")
repro.pop("duration")
# import Stimuli
mtags = ut.find_mtags_for_tag(b, t)
stim_index = 0
for mt in mtags:
mtags, positions = ut.find_mtags_for_tag(b, t)
for i, mt in enumerate(mtags):
mt_positions = np.atleast_2d(mt.positions[:]).T
mt_extents = np.atleast_2d(mt.extents[:]).T
for i in range(mt.positions.shape[0]):
settings = ut.mtag_settings_to_yaml(mt, i)
stim_start = mt_positions[i, 0]
stim_duration = mt_extents[i, 0]
for p in positions[i]:
settings = ut.mtag_settings_to_yaml(mt, p)
stim_start = mt_positions[p, 0]
stim_duration = mt_extents[p, 0]
stim = Stimulus.get_template_tuple()
stim["stimulus_id"] = str(uuid.uuid1())
stim["stimulus_index"] = stim_index
stim["stimulus_index"] = p
stim["start"] = stim_start
stim["duration"] = stim_duration
stim["settings"] = settings
stim["mtag_id"] = mt.id
stim["stimulus_name"] = mt.name
stim.update(repro)
Stimulus.insert1(stim, skip_duplicates=True)
stim_index += 1
f.close()
f = None
else:
def scan_folder_for_repros(dataset):
pass
def populate_repros(data_path):
print("\tImporting RePro(s) of %s" % data_path)
dset_name = os.path.split(data_path)[-1]
if len(Dataset & {"dataset_id": dset_name}) != 1:
return False
dataset = dict((Dataset & {"dataset_id": dset_name}).fetch1())
if dataset["has_nix"]:
scan_nix_file_for_repros(dataset)
else:
scan_folder_for_repros(dataset)
return True
@ -403,8 +416,9 @@ def populate(datasets):
if __name__ == "__main__":
data_dir = "../../science/high_frequency_chirps/data"
data_dir = "/data/apteronotus"
# data_dir = "../high_freq_chirps/data"
datasets = glob.glob(os.path.join(data_dir, '2018-11-20*'))
drop_tables()
# populate(datasets)
datasets = glob.glob(os.path.join(data_dir, '2018-03-22-ak-*'))
# drop_tables()
populate(datasets)

28
util.py
View File

@ -104,22 +104,32 @@ def nix_metadata_to_yaml(section, cur_depth=0, val_count=1):
def find_mtags_for_tag(block, tag):
"""
Finds those multi tags and the respective positions within that match to a certain
repro run.
@:returns list of mtags, list of mtag positions
"""
assert(isinstance(block, nix.pycore.block.Block))
assert(isinstance(tag, nix.pycore.tag.Tag))
mtags = []
indices = []
tag_start = np.atleast_1d(tag.position)
tag_end = tag_start + np.atleast_1d(tag.extent)
for mt in block.multi_tags:
mt_start = np.atleast_1d(mt.positions[0, :])
mt_end = np.atleast_1d(mt.positions[-1, :] + mt.extents[-1, :])
in_tag = True
for i in range(len(tag_start)):
if mt_start[i] < tag_start[i] or mt_end[i] > tag_end[i]:
in_tag = False
break
if in_tag:
position_count = mt.positions.shape[0]
in_tag_positions = []
for i in range(position_count):
mt_start = np.atleast_1d(mt.positions[i, :])
mt_end = mt_start + np.atleast_1d(mt.extents[i, :])
for j in range(len(tag_start)):
if mt_start[j] >= tag_start[j] and mt_end[j] <= tag_end[j]:
in_tag_positions.append(i)
if len(in_tag_positions) > 0:
mtags.append(mt)
return mtags
indices.append(in_tag_positions)
return mtags, indices
def mtag_settings_to_yaml(mtag, pos_index):