[utils] populating rePro

database.py

@@ -3,7 +3,8 @@ import datajoint as dj
 import nixio as nix
 import os
 import glob
-from util import read_info_file, find_key_recursive, deep_get, read_dataset_info
+from util import read_info_file, find_key_recursive, deep_get, read_dataset_info, \
+    nix_metadata_to_yaml
 from IPython import embed
 
 schema = dj.schema("fish_book", locals())
@@ -162,6 +163,26 @@ class CellDatasetMap(dj.Manual):
     """
 
 
+@schema
+class RePro(dj.Manual):
+    definition = """
+    repro_id : varchar(512)
+    run : smallint
+    -> Dataset
+    ----
+    repro_name : varchar(512)
+    settings : varchar(3000)
+    """
+
+    @staticmethod
+    def get_template_tuple(repro_id=None):
+        tup = dict(repro_id=None, dataset_id=None, run=0, repro_name="", settings="")
+        if repro_id is not None:
+            d = dict((RePro() & {"repro_id": repro_id}).fetch1())
+            return d
+        return tup
+
+
 def populate_datasets(data_path):
     print("Importing dataset %s" % data_path)
     if not os.path.exists(data_path):
@@ -270,6 +291,32 @@ def populate_cells(data_path):
     CellDatasetMap.insert1(mm, skip_duplicates=True)
 
 
+def populate_repros(data_path):
+    print("Importing RePro(s) of %s" % data_path)
+    dset_name = os.path.split(data_path)[-1]
+    dataset = dict((Dataset & {"dataset_id": dset_name}).fetch1())
+
+    if dataset["has_nix"]:
+        print("scanning nix file")
+        nix_files = glob.glob(os.path.join(dataset["data_source"], "*.nix"))
+        for nf in nix_files:
+            f = nix.File.open(nf, nix.FileMode.ReadOnly)
+            b = f.blocks[0]
+            for t in b.tags:
+                if "relacs.repro_run" in t.type and "RePro-Info" in t.metadata.sections:
+                    rp = RePro.get_template_tuple()
+                    rp["run"] = t.metadata["RePro-Info"]["Run"]
+                    rp["dataset_id"] = dataset["dataset_id"]
+                    rp["repro_id"] = t.name
+                    rp["repro_name"] = t.metadata["RePro-Info"]["RePro"]
+                    rp["settings"] = nix_metadata_to_yaml(t.metadata.sections[0])
+                    RePro.insert1(rp, skip_duplicates=True)
+            f.close()
+            f = None
+    else:
+        pass
+
+
 def drop_tables():
     Dataset.drop()
     Subject.drop()
@@ -277,9 +324,10 @@ def drop_tables():
 
 def populate(datasets):
     for d in datasets:
-        populate_datasets(d)
+        #populate_datasets(d)
-        populate_subjects(d)
+        #populate_subjects(d)
-        populate_cells(d)
+        #populate_cells(d)
+        populate_repros(d)
 
 
 if __name__ == "__main__":

util.py

@@ -1,7 +1,9 @@
 from functools import reduce
+import nixio as nix
 import os
 import glob
 import datetime as dt
+from IPython import embed
 
 
 def read_info_file(file_name):
@@ -75,3 +77,38 @@ def read_dataset_info(info_file):
 
     return exp, rec_date, quality, comment, has_nix
 
+
+def nix_metadata_to_dict(section):
+    info = {}
+    for p in section.props:
+        info[p.name] = [v.value for v in p.values]
+    for s in section.sections:
+        info[s.name] = nix_metadata_to_dict(s)
+    return info
+
+
+def nix_metadata_to_yaml(section, cur_depth=0, val_count=1):
+    assert(isinstance(section, nix.section.SectionMixin))
+    yaml = "%s%s:\n" % ("\t" * cur_depth, section.name)
+    for p in section.props:
+        val_str = ""
+        if val_count > 1 and len(p.values) > 1:
+            val_str = "[" + ', '.join([v.to_string() for v in p.values]) + "]"
+        elif len(p.values) == 1:
+            val_str = p.values[0].to_string()
+        yaml += "%s%s: %s\n" % ("\t" * (cur_depth+1), p.name, val_str)
+    for s in section.sections:
+        yaml += nix_metadata_to_yaml(s, cur_depth+1)
+    return yaml
+
+
+if __name__ == "__main__":
+    nix_file = "../../science/high_frequency_chirps/data/2018-11-09-aa-invivo-1/2018-11-09-aa-invivo-1.nix"
+    f = nix.File.open(nix_file, nix.FileMode.ReadOnly)
+    b = f.blocks[0]
+    yml = nix_metadata_to_yaml(b.tags[0].metadata)
+    print(yml)
+    print("-"* 80)
+    print(nix_metadata_to_yaml(b.metadata))
+    f.close()
+