fishbook/database.py

import numpy as np
import datajoint as dj
import nixio as nix
import os
import glob
from util import read_info_file, find_key_recursive, deep_get
from IPython import embed
import datetime as dt

data_dir = 'data'
schema = dj.schema("fish_book", locals())


@schema
class Dataset(dj.Manual):
    definition = """ # Dataset
       dataset_id : varchar(256)
       ----
       data_source : varchar(512) # path to the dataset
       experimenter : varchar(512) 
       recording_date : date
       has_nix : bool
       """

    @staticmethod
    def get_template_tuple(id=None):
        if id is not None:
            d = dict((Dataset() & {"dataset_id": id}).fetch1())
            return d
        return dict(dataset_id=None, data_source="", experimenter="", recording_date=None, has_nix=False)

    @staticmethod
    def get_nix_file(key):
        dset = (Dataset() & key).fetch1()
        if dset["ignore"]:
            return None
        file_path = os.path.join(dset["data_source"], dset["dataset_id"] + ".nix")
        if not (os.path.exists(file_path)):
            print("\t No nix file found for path: %s" % dset["data_source"])
            return None
        if not Dataset.check_file_integrity(file_path):
            return None
        return file_path

    @staticmethod
    def check_file_integrity(nix_file):
        sane = True
        try:
            f = nix.File.open(nix_file, nix.FileMode.ReadOnly)
            b = f.blocks[0]
            m = b.metadata
            if "Recording" not in m.sections:
                Warning("\t Could not find Recording section in dataset: %s" % nix_file)
                sane = False
            f.close()
        except ():
            print("file: %s is NOT SANE!")
            sane = False
        return sane



@schema
class Subject(dj.Manual):
    definition = """
    # Subject
    subject_id : varchar(256)
    ----
    species : varchar(256) 
    """

    @staticmethod
    def get_template_tuple(subject_id=None):
        tup = dict(subject_id=None, species="")
        if subject_id is not None:
            d = dict((Subject() & {"subject_id": subject_id}).fetch1())
            return d
        return tup

    def make(self, key):
        file_path = Dataset.get_nix_file(key)
        if file_path is None:
            return
        nix_file = nix.File.open(file_path, nix.FileMode.ReadOnly)
        m = nix_file.blocks[0].metadata
        inserts = Subject.get_template_tuple()
        subj_info = m["Recording"]["Subject"]
        inserts["subject_id"] = subj_info["Identifier"]
        inserts["species"] = subj_info["Species"][0]
        inserts["weight"] = subj_info["Weight"]
        inserts["size"] = subj_info["Size"]
        inserts["eod_frequency"] = np.round(subj_info["EOD Frequency"] * 10) / 10
        inserts.update(key)
        self.insert1(inserts, skip_duplicates=True)
        nix_file.close()


@schema
class SubjectDatasetMap(dj.Manual):
    definition = """
    # SubjectDatasetMap
    -> Subject
    -> Dataset
    """


@schema
class SubjectProperties(dj.Manual):
    definition = """
    # SubjectProperties
    id : int auto_increment
    ---- 
    -> Subject
    recording_date : date
    weight : float
    size : float
    eod_frequency : float
    """

    def get_template_tuple(id=None):
        tup = dict(id=None, subject_id=None, recording_date=None, weight=0.0, size=0.0, eod_frequency=0.0)
        if id is not None:
            return dict((SubjectProperties() & {"id": id}).fetch1())
        return tup


def read_info(info_file):
    if not os.path.exists(info_file):
        return None, None, False
    has_nix = len(glob.glob(os.path.sep.join(info_file.split(os.path.sep)[:-1]) + os.path.sep + "*.nix")) > 0
    info = read_info_file(info_file)
    p = []
    find_key_recursive(info, "Experimenter", p)
    if len(p) > 0:
        exp = deep_get(info, p)
    p = []
    find_key_recursive(info, "Date", p)
    if len(p) > 0:
        rec_date = dt.date.fromisoformat(deep_get(info, p))
    return exp, rec_date, has_nix


def populate_datasets(data_path):
    print("Importing dataset %s" % data_path)
    if not os.path.exists(data_path):
        return
    dset_name = os.path.split(data_path)[-1]
    experimenter, rec_date, has_nix = read_info(os.path.join(data_path, 'info.dat'))
    if not experimenter:
        return

    inserts = Dataset.get_template_tuple()
    inserts["dataset_id"] = dset_name
    inserts["data_source"] = data_path
    inserts["experimenter"] = experimenter
    inserts["recording_date"] = rec_date
    inserts["has_nix"] = has_nix

    Dataset().insert1(inserts, skip_duplicates=True)


def populate_subjects(data_path):
    print("Importing subject(s) of %s" % data_path)
    dset_name = os.path.split(data_path)[-1]
    info_file = os.path.join(data_path, 'info.dat')
    if not os.path.exists(info_file):
        return None, None, False
    info = read_info_file(info_file)
    p = []
    find_key_recursive(info, "Subject", p)
    if len(p) > 0:
        subj = deep_get(info, p)
    inserts = Subject.get_template_tuple()
    inserts["subject_id"] = subj["Identifier"]
    inserts["species"] = subj["Species"]
    Subject().insert1(inserts, skip_duplicates=True)

    # multi mach entry
    dataset = dict((Dataset() & {"dataset_id": dset_name}).fetch1())
    mm = dict(dataset_id=dataset["dataset_id"], subject_id=subj["Identifier"])
    SubjectDatasetMap.insert1(mm, skip_duplicates=True)

    # subject properties
    props = SubjectProperties.get_template_tuple()
    props["subject_id"] = subj["Identifier"]
    props["recording_date"] = dataset["recording_date"]
    if "Weight" in subj.keys():
        props["weight"] = np.round(float(subj["Weight"][:-1]), 1)
    if "Size" in subj.keys():
        props["size"] = np.round(float(subj["Size"][:-2]), 1)
    if "EOD Frequency" in subj.keys():
        props["eod_frequency"] = np.round(float(subj["EOD Frequency"][:-2]))
    p = props.copy()
    p.pop("id")
    if len(SubjectProperties & p) == 0:
        SubjectProperties.insert1(props, skip_duplicates=True)


if __name__ == "__main__":
    datasets = glob.glob('/data/apteronotus/2018-05-08*')
    # Dataset.drop()
    # Subject.drop()
    # SubjectProperties.drop()
    # SubjectDatasetMap.drop()
    for d in datasets:
        populate_datasets(d)
        populate_subjects(d)