diff --git a/database.py b/database.py index c2673e2..c1d3560 100644 --- a/database.py +++ b/database.py @@ -5,6 +5,7 @@ import os import glob import util as ut import uuid +import pyrelacs from IPython import embed @@ -195,6 +196,7 @@ class Stimulus(dj.Manual): --- stimulus_index : int stimulus_name : varchar(512) + mtag_id : varchar(50) start : float duration : float settings : varchar(3000) @@ -246,7 +248,7 @@ def populate_subjects(data_path): inserts["species"] = subj["Species"] Subject().insert1(inserts, skip_duplicates=True) - # multi mach entry + # multi match entry dataset = dict((Dataset() & {"dataset_id": dset_name}).fetch1()) mm = dict(dataset_id=dataset["dataset_id"], subject_id=subj["Identifier"]) SubjectDatasetMap.insert1(mm, skip_duplicates=True) @@ -318,6 +320,70 @@ def populate_cells(data_path): CellDatasetMap.insert1(mm, skip_duplicates=True) +def scan_nix_file_for_repros(dataset): + print("\t\tscanning nix file") + nix_files = glob.glob(os.path.join(dataset["data_source"], "*.nix")) + for nf in nix_files: + if not Dataset.check_file_integrity(nf): + print("file is not sane!!!") + continue + f = nix.File.open(nf, nix.FileMode.ReadOnly) + b = f.blocks[0] + repro_runs = [t for t in b.tags if "relacs.repro_run" in t.type] + for t in repro_runs: + rs = t.metadata.find_sections(lambda x: "Run" in x.props) + if len(rs) == 0: + continue + rs = rs[0] + rp = Repro.get_template_tuple() + rp["run"] = rs["Run"] + rp["repro_name"] = rs["RePro"] + rp["dataset_id"] = dataset["dataset_id"] + rp["repro_id"] = t.name + settings = t.metadata.find_sections(lambda x: "settings" in x.type) + if len(settings) > 0: + rp["settings"] = ut.nix_metadata_to_yaml(settings[0]) + else: + rp["settings"] = ut.nix_metadata_to_yaml(t.metadata) + rp["start"] = t.position[0] + rp["duration"] = t.extent[0] + Repro.insert1(rp, skip_duplicates=True) + + # import Stimuli + repro = dict((Repro & dict(repro_id=rp["repro_id"], dataset_id=rp["dataset_id"])).fetch1()) + repro.pop("settings") + repro.pop("repro_name") + repro.pop("start") + repro.pop("duration") + + mtags, positions = ut.find_mtags_for_tag(b, t) + for i, mt in enumerate(mtags): + mt_positions = np.atleast_2d(mt.positions[:]).T + mt_extents = np.atleast_2d(mt.extents[:]).T + for p in positions[i]: + settings = ut.mtag_settings_to_yaml(mt, p) + stim_start = mt_positions[p, 0] + stim_duration = mt_extents[p, 0] + + stim = Stimulus.get_template_tuple() + stim["stimulus_id"] = str(uuid.uuid1()) + stim["stimulus_index"] = p + stim["start"] = stim_start + stim["duration"] = stim_duration + stim["settings"] = settings + stim["mtag_id"] = mt.id + stim["stimulus_name"] = mt.name + stim.update(repro) + Stimulus.insert1(stim, skip_duplicates=True) + f.close() + f = None + + +def scan_folder_for_repros(dataset): + + pass + + def populate_repros(data_path): print("\tImporting RePro(s) of %s" % data_path) dset_name = os.path.split(data_path)[-1] @@ -326,62 +392,9 @@ def populate_repros(data_path): dataset = dict((Dataset & {"dataset_id": dset_name}).fetch1()) if dataset["has_nix"]: - print("\t\tscanning nix file") - nix_files = glob.glob(os.path.join(dataset["data_source"], "*.nix")) - for nf in nix_files: - if not Dataset.check_file_integrity(nf): - print("file is not sane!!!") - continue - f = nix.File.open(nf, nix.FileMode.ReadOnly) - b = f.blocks[0] - for t in b.tags: - if "relacs.repro_run" in t.type: - rs = t.metadata.find_sections(lambda x: "Run" in x.props) - if len(rs) == 0: - continue - rs = rs[0] - rp = Repro.get_template_tuple() - rp["run"] = rs["Run"] - rp["repro_name"] = rs["RePro"] - rp["dataset_id"] = dataset["dataset_id"] - rp["repro_id"] = t.name - settings = t.metadata.find_sections(lambda x: "settings" in x.type) - if len(settings) > 0: - rp["settings"] = ut.nix_metadata_to_yaml(settings[0]) - else: - rp["settings"] = ut.nix_metadata_to_yaml(t.metadata) - rp["start"] = t.position[0] - rp["duration"] = t.extent[0] - Repro.insert1(rp, skip_duplicates=True) - repro = dict((Repro & dict(repro_id="BaselineActivity_2", dataset_id="2018-11-20-ae-invivo-1")).fetch1()) - repro.pop("settings") - repro.pop("repro_name") - repro.pop("start") - repro.pop("duration") - # import Stimuli - mtags = ut.find_mtags_for_tag(b, t) - stim_index = 0 - for mt in mtags: - mt_positions = np.atleast_2d(mt.positions[:]).T - mt_extents = np.atleast_2d(mt.extents[:]).T - for i in range(mt.positions.shape[0]): - settings = ut.mtag_settings_to_yaml(mt, i) - stim_start = mt_positions[i, 0] - stim_duration = mt_extents[i, 0] - - stim = Stimulus.get_template_tuple() - stim["stimulus_id"] = str(uuid.uuid1()) - stim["stimulus_index"] = stim_index - stim["start"] = stim_start - stim["duration"] = stim_duration - stim["settings"] = settings - stim.update(repro) - Stimulus.insert1(stim, skip_duplicates=True) - stim_index += 1 - f.close() - f = None + scan_nix_file_for_repros(dataset) else: - pass + scan_folder_for_repros(dataset) return True @@ -403,8 +416,9 @@ def populate(datasets): if __name__ == "__main__": - data_dir = "../../science/high_frequency_chirps/data" + data_dir = "/data/apteronotus" # data_dir = "../high_freq_chirps/data" - datasets = glob.glob(os.path.join(data_dir, '2018-11-20*')) - drop_tables() - # populate(datasets) + datasets = glob.glob(os.path.join(data_dir, '2018-03-22-ak-*')) + # drop_tables() + populate(datasets) + diff --git a/util.py b/util.py index 6ce4e1d..6716409 100644 --- a/util.py +++ b/util.py @@ -104,22 +104,32 @@ def nix_metadata_to_yaml(section, cur_depth=0, val_count=1): def find_mtags_for_tag(block, tag): + """ + Finds those multi tags and the respective positions within that match to a certain + repro run. + + @:returns list of mtags, list of mtag positions + """ assert(isinstance(block, nix.pycore.block.Block)) assert(isinstance(tag, nix.pycore.tag.Tag)) mtags = [] + indices = [] tag_start = np.atleast_1d(tag.position) tag_end = tag_start + np.atleast_1d(tag.extent) for mt in block.multi_tags: - mt_start = np.atleast_1d(mt.positions[0, :]) - mt_end = np.atleast_1d(mt.positions[-1, :] + mt.extents[-1, :]) - in_tag = True - for i in range(len(tag_start)): - if mt_start[i] < tag_start[i] or mt_end[i] > tag_end[i]: - in_tag = False - break - if in_tag: + position_count = mt.positions.shape[0] + in_tag_positions = [] + for i in range(position_count): + mt_start = np.atleast_1d(mt.positions[i, :]) + mt_end = mt_start + np.atleast_1d(mt.extents[i, :]) + + for j in range(len(tag_start)): + if mt_start[j] >= tag_start[j] and mt_end[j] <= tag_end[j]: + in_tag_positions.append(i) + if len(in_tag_positions) > 0: mtags.append(mt) - return mtags + indices.append(in_tag_positions) + return mtags, indices def mtag_settings_to_yaml(mtag, pos_index): @@ -137,9 +147,9 @@ def mtag_settings_to_yaml(mtag, pos_index): feat_name = feat.data.label if feat.data.label and len(feat.data.label) > 0 else feat.data.name feat_unit = feat.data.unit if feat.data.unit and len(feat.data.unit) > 0 else "" if feat_data.shape[0] == 1: - feat_content = "%s %s" % (feat_data[0], feat_unit) + feat_content = "%s %s" % (feat_data[0], feat_unit) else: - feat_content = "[" + ','.join(map(str, feat_data[:])) + "] %s" % feat_unit + feat_content = "[" + ','.join(map(str, feat_data[:])) + "] %s" % feat_unit yaml += "\t%s: %s\n" % (feat_name, feat_content) return yaml