forked from jgrewe/fishbook
kind of works for nix files
This commit is contained in:
parent
dee40d879c
commit
eba710b615
134
database.py
134
database.py
@ -5,6 +5,7 @@ import os
|
||||
import glob
|
||||
import util as ut
|
||||
import uuid
|
||||
import pyrelacs
|
||||
|
||||
from IPython import embed
|
||||
|
||||
@ -195,6 +196,7 @@ class Stimulus(dj.Manual):
|
||||
---
|
||||
stimulus_index : int
|
||||
stimulus_name : varchar(512)
|
||||
mtag_id : varchar(50)
|
||||
start : float
|
||||
duration : float
|
||||
settings : varchar(3000)
|
||||
@ -246,7 +248,7 @@ def populate_subjects(data_path):
|
||||
inserts["species"] = subj["Species"]
|
||||
Subject().insert1(inserts, skip_duplicates=True)
|
||||
|
||||
# multi mach entry
|
||||
# multi match entry
|
||||
dataset = dict((Dataset() & {"dataset_id": dset_name}).fetch1())
|
||||
mm = dict(dataset_id=dataset["dataset_id"], subject_id=subj["Identifier"])
|
||||
SubjectDatasetMap.insert1(mm, skip_duplicates=True)
|
||||
@ -318,6 +320,70 @@ def populate_cells(data_path):
|
||||
CellDatasetMap.insert1(mm, skip_duplicates=True)
|
||||
|
||||
|
||||
def scan_nix_file_for_repros(dataset):
|
||||
print("\t\tscanning nix file")
|
||||
nix_files = glob.glob(os.path.join(dataset["data_source"], "*.nix"))
|
||||
for nf in nix_files:
|
||||
if not Dataset.check_file_integrity(nf):
|
||||
print("file is not sane!!!")
|
||||
continue
|
||||
f = nix.File.open(nf, nix.FileMode.ReadOnly)
|
||||
b = f.blocks[0]
|
||||
repro_runs = [t for t in b.tags if "relacs.repro_run" in t.type]
|
||||
for t in repro_runs:
|
||||
rs = t.metadata.find_sections(lambda x: "Run" in x.props)
|
||||
if len(rs) == 0:
|
||||
continue
|
||||
rs = rs[0]
|
||||
rp = Repro.get_template_tuple()
|
||||
rp["run"] = rs["Run"]
|
||||
rp["repro_name"] = rs["RePro"]
|
||||
rp["dataset_id"] = dataset["dataset_id"]
|
||||
rp["repro_id"] = t.name
|
||||
settings = t.metadata.find_sections(lambda x: "settings" in x.type)
|
||||
if len(settings) > 0:
|
||||
rp["settings"] = ut.nix_metadata_to_yaml(settings[0])
|
||||
else:
|
||||
rp["settings"] = ut.nix_metadata_to_yaml(t.metadata)
|
||||
rp["start"] = t.position[0]
|
||||
rp["duration"] = t.extent[0]
|
||||
Repro.insert1(rp, skip_duplicates=True)
|
||||
|
||||
# import Stimuli
|
||||
repro = dict((Repro & dict(repro_id=rp["repro_id"], dataset_id=rp["dataset_id"])).fetch1())
|
||||
repro.pop("settings")
|
||||
repro.pop("repro_name")
|
||||
repro.pop("start")
|
||||
repro.pop("duration")
|
||||
|
||||
mtags, positions = ut.find_mtags_for_tag(b, t)
|
||||
for i, mt in enumerate(mtags):
|
||||
mt_positions = np.atleast_2d(mt.positions[:]).T
|
||||
mt_extents = np.atleast_2d(mt.extents[:]).T
|
||||
for p in positions[i]:
|
||||
settings = ut.mtag_settings_to_yaml(mt, p)
|
||||
stim_start = mt_positions[p, 0]
|
||||
stim_duration = mt_extents[p, 0]
|
||||
|
||||
stim = Stimulus.get_template_tuple()
|
||||
stim["stimulus_id"] = str(uuid.uuid1())
|
||||
stim["stimulus_index"] = p
|
||||
stim["start"] = stim_start
|
||||
stim["duration"] = stim_duration
|
||||
stim["settings"] = settings
|
||||
stim["mtag_id"] = mt.id
|
||||
stim["stimulus_name"] = mt.name
|
||||
stim.update(repro)
|
||||
Stimulus.insert1(stim, skip_duplicates=True)
|
||||
f.close()
|
||||
f = None
|
||||
|
||||
|
||||
def scan_folder_for_repros(dataset):
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def populate_repros(data_path):
|
||||
print("\tImporting RePro(s) of %s" % data_path)
|
||||
dset_name = os.path.split(data_path)[-1]
|
||||
@ -326,62 +392,9 @@ def populate_repros(data_path):
|
||||
dataset = dict((Dataset & {"dataset_id": dset_name}).fetch1())
|
||||
|
||||
if dataset["has_nix"]:
|
||||
print("\t\tscanning nix file")
|
||||
nix_files = glob.glob(os.path.join(dataset["data_source"], "*.nix"))
|
||||
for nf in nix_files:
|
||||
if not Dataset.check_file_integrity(nf):
|
||||
print("file is not sane!!!")
|
||||
continue
|
||||
f = nix.File.open(nf, nix.FileMode.ReadOnly)
|
||||
b = f.blocks[0]
|
||||
for t in b.tags:
|
||||
if "relacs.repro_run" in t.type:
|
||||
rs = t.metadata.find_sections(lambda x: "Run" in x.props)
|
||||
if len(rs) == 0:
|
||||
continue
|
||||
rs = rs[0]
|
||||
rp = Repro.get_template_tuple()
|
||||
rp["run"] = rs["Run"]
|
||||
rp["repro_name"] = rs["RePro"]
|
||||
rp["dataset_id"] = dataset["dataset_id"]
|
||||
rp["repro_id"] = t.name
|
||||
settings = t.metadata.find_sections(lambda x: "settings" in x.type)
|
||||
if len(settings) > 0:
|
||||
rp["settings"] = ut.nix_metadata_to_yaml(settings[0])
|
||||
else:
|
||||
rp["settings"] = ut.nix_metadata_to_yaml(t.metadata)
|
||||
rp["start"] = t.position[0]
|
||||
rp["duration"] = t.extent[0]
|
||||
Repro.insert1(rp, skip_duplicates=True)
|
||||
repro = dict((Repro & dict(repro_id="BaselineActivity_2", dataset_id="2018-11-20-ae-invivo-1")).fetch1())
|
||||
repro.pop("settings")
|
||||
repro.pop("repro_name")
|
||||
repro.pop("start")
|
||||
repro.pop("duration")
|
||||
# import Stimuli
|
||||
mtags = ut.find_mtags_for_tag(b, t)
|
||||
stim_index = 0
|
||||
for mt in mtags:
|
||||
mt_positions = np.atleast_2d(mt.positions[:]).T
|
||||
mt_extents = np.atleast_2d(mt.extents[:]).T
|
||||
for i in range(mt.positions.shape[0]):
|
||||
settings = ut.mtag_settings_to_yaml(mt, i)
|
||||
stim_start = mt_positions[i, 0]
|
||||
stim_duration = mt_extents[i, 0]
|
||||
|
||||
stim = Stimulus.get_template_tuple()
|
||||
stim["stimulus_id"] = str(uuid.uuid1())
|
||||
stim["stimulus_index"] = stim_index
|
||||
stim["start"] = stim_start
|
||||
stim["duration"] = stim_duration
|
||||
stim["settings"] = settings
|
||||
stim.update(repro)
|
||||
Stimulus.insert1(stim, skip_duplicates=True)
|
||||
stim_index += 1
|
||||
f.close()
|
||||
f = None
|
||||
scan_nix_file_for_repros(dataset)
|
||||
else:
|
||||
pass
|
||||
scan_folder_for_repros(dataset)
|
||||
return True
|
||||
|
||||
|
||||
@ -403,8 +416,9 @@ def populate(datasets):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
data_dir = "../../science/high_frequency_chirps/data"
|
||||
data_dir = "/data/apteronotus"
|
||||
# data_dir = "../high_freq_chirps/data"
|
||||
datasets = glob.glob(os.path.join(data_dir, '2018-11-20*'))
|
||||
drop_tables()
|
||||
# populate(datasets)
|
||||
datasets = glob.glob(os.path.join(data_dir, '2018-03-22-ak-*'))
|
||||
# drop_tables()
|
||||
populate(datasets)
|
||||
|
||||
|
||||
32
util.py
32
util.py
@ -104,22 +104,32 @@ def nix_metadata_to_yaml(section, cur_depth=0, val_count=1):
|
||||
|
||||
|
||||
def find_mtags_for_tag(block, tag):
|
||||
"""
|
||||
Finds those multi tags and the respective positions within that match to a certain
|
||||
repro run.
|
||||
|
||||
@:returns list of mtags, list of mtag positions
|
||||
"""
|
||||
assert(isinstance(block, nix.pycore.block.Block))
|
||||
assert(isinstance(tag, nix.pycore.tag.Tag))
|
||||
mtags = []
|
||||
indices = []
|
||||
tag_start = np.atleast_1d(tag.position)
|
||||
tag_end = tag_start + np.atleast_1d(tag.extent)
|
||||
for mt in block.multi_tags:
|
||||
mt_start = np.atleast_1d(mt.positions[0, :])
|
||||
mt_end = np.atleast_1d(mt.positions[-1, :] + mt.extents[-1, :])
|
||||
in_tag = True
|
||||
for i in range(len(tag_start)):
|
||||
if mt_start[i] < tag_start[i] or mt_end[i] > tag_end[i]:
|
||||
in_tag = False
|
||||
break
|
||||
if in_tag:
|
||||
position_count = mt.positions.shape[0]
|
||||
in_tag_positions = []
|
||||
for i in range(position_count):
|
||||
mt_start = np.atleast_1d(mt.positions[i, :])
|
||||
mt_end = mt_start + np.atleast_1d(mt.extents[i, :])
|
||||
|
||||
for j in range(len(tag_start)):
|
||||
if mt_start[j] >= tag_start[j] and mt_end[j] <= tag_end[j]:
|
||||
in_tag_positions.append(i)
|
||||
if len(in_tag_positions) > 0:
|
||||
mtags.append(mt)
|
||||
return mtags
|
||||
indices.append(in_tag_positions)
|
||||
return mtags, indices
|
||||
|
||||
|
||||
def mtag_settings_to_yaml(mtag, pos_index):
|
||||
@ -137,9 +147,9 @@ def mtag_settings_to_yaml(mtag, pos_index):
|
||||
feat_name = feat.data.label if feat.data.label and len(feat.data.label) > 0 else feat.data.name
|
||||
feat_unit = feat.data.unit if feat.data.unit and len(feat.data.unit) > 0 else ""
|
||||
if feat_data.shape[0] == 1:
|
||||
feat_content = "%s %s" % (feat_data[0], feat_unit)
|
||||
feat_content = "%s %s" % (feat_data[0], feat_unit)
|
||||
else:
|
||||
feat_content = "[" + ','.join(map(str, feat_data[:])) + "] %s" % feat_unit
|
||||
feat_content = "[" + ','.join(map(str, feat_data[:])) + "] %s" % feat_unit
|
||||
yaml += "\t%s: %s\n" % (feat_name, feat_content)
|
||||
return yaml
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user