from fishbook.backend.database import Cells, Datasets, CellDatasetMap, Subjects, SubjectProperties, SubjectDatasetMap, Stimuli, Repros from .util import safe_get_val, results_check import nixio as nix import os import numpy as np from IPython import embed from fishbook.backend.util import progress import datetime as dt import yaml class Cell: """The Cell class represents a recorded cell. It is characterized by *id*, the cell *type*, the *firing_rate*, and the recording *location*. """ def __init__(self, cell_id=None, tuple=None): if tuple: self.__tuple = tuple elif cell_id: pattern = "cell_id like '{0:s}'".format(cell_id) cells = (Cells & pattern) results_check(cells, cell_id, "Cell ID") self.__tuple = cells.fetch(as_dict=True)[0] else: print("Empty Cell, not linked to any database entry!") @property def id(self): """The id of the cell. Unsually the same as the dataset name. Returns: str: the id """ return self.__tuple["cell_id"] if "cell_id" in self.__tuple.keys() else "" @property def type(self): """The cell type. Returns: str: the cell type """ return self.__tuple["cell_type"] if "cell_type" in self.__tuple.keys() else "" @property def firing_rate(self): """The firing rate of the neuron, if applicable, else 0.0. Returns: float: the firing rate. """ return self.__tuple["firing_rate"] if "firing_rate" in self.__tuple.keys() else 0.0 @property def location(self): """The recording location which consists of the structure (e.g. brain), the region (e.g. ELL), the subregion (e.g. lateral segment), the depth below surface, the lateral position and the transversal section if applicable. The latter two are given according to the [Apteronotus brain atlas](http://nelson.beckman.illinois.edu/atlas.html). Returns: dict: the recording location """ keys = ["structure", "region", "subregion", "depth", "lateral_pos", "transversal_section"] loc = {} for k in keys: if k in self.__tuple.keys(): loc[k] = self.__tuple[k] else: loc[k] = "" return loc @property def subject(self): """The subject in which this cell was recorded. Returns: fishbook.Subject: the subject """ return Subject(tuple=(Subjects & {"subject_id": self.__tuple["subject_id"]}).fetch(limit=1, as_dict=True)[0]) @property def datasets(self): """All datasets that have been collected from this cell. Returns: list of fishbook.Dataset: the linked datasets """ return [Dataset(tuple=d) for d in (Datasets & (CellDatasetMap & {"cell_id": self.id})).fetch(as_dict=True)] @property def repro_runs(self): """All relacs RePros that have been run while recording this cell. Returns: list of fishbook.RePro: the repros """ repros = (Repros & "cell_id = '%s'" % self.id) return [RePro(tuple=r) for r in repros] @staticmethod def unique_cell_types(): """List of unique cell types stored in the database. Returns: np.array of str: the cell types """ return np.unique(Cells.fetch("cell_type")) @staticmethod def find(cell_type=None, species=None, quality="good", test=False): """Find cells that match with respect to cell type, subject species, and recording quality. Args: cell_type (str, optional): The desired cell type, case insensitive. Defaults to None. species (str, optional): The species name, or parts of it. Defaults to None. quality (str, optional): The desired qulality. Defaults to "good". test (bool, optional): Whether or not this query is a test run. Defaults to False. Returns: list of fishbook.Cell: the matching cells, empty it test==True. int : the number of matches """ cs = Cells * CellDatasetMap * Datasets * Subjects if cell_type: cs = cs & "cell_type like '%{0:s}%'".format(cell_type) if species: cs = cs & "species like '%{0:s}%'".format(species) if quality: cs = cs & "quality like '%{0:s}%'".format(quality) results = [] total = len(cs) if not test: for i, c in enumerate(cs): progress(i+1, total, "fetching %i matches" % total) results.append(Cell(tuple=c)) return results, total def __str__(self): str = "" str += "Cell: %s \t type: %s\n" % (self.id, self.type) str += "Baseline firing: %.3f\n" % self.firing_rate str += "structure: %s \t region: %s\t subregion: %s\n" % (self.__tuple["structure"], self.__tuple["region"], self.__tuple["subregion"]) return str @property def _tuple(self): return self.__tuple.copy() class Dataset: """ The Dataset class represents an entry in the "datasets" table in the database. In the relacs context a Dataset represents all that has been recorded between pressing enter. For example all that is stored in the folder 2020-01-01-aa. Dataset contains several basic properties and allows access to associated cells and subjects. """ def __init__(self, dataset_id=None, tuple=None): """Constructor of a Dataset entity. Args: dataset_id (str, optional): the id. Defaults to None. tuple (tuple, optional): The properties used to identify the entity in the database. Usually only for internal use. Defaults to None. """ self.__samplerate = 0.0 if tuple: self.__tuple = tuple elif dataset_id: pattern = "dataset_id like '%{0:s}%'".format(dataset_id) dsets = (Datasets & pattern) results_check(dsets, dataset_id, "Dataset ID") self.__tuple = dsets.fetch(limit=1, as_dict=True)[0] else: print("Empty dataset, not linked to any database entry!") if len(self.__tuple.keys()) > 0: self.__find_samplerate() @property def id(self): """The id of the dataset. Returns: str: the id """ return self.__tuple["dataset_id"] @property def experimenter(self): """The person who did record the dataset. Returns: str: The experimenter """ return self.__tuple["experimenter"] @property def recording_date(self): """The Dataset's recording date. Returns: str: the recording date. """ return self.__tuple["recording_date"] @property def recording_duration(self): """Recording duration. Returns: float: recprding duration given in seconds. """ return self.__tuple["duration"] @property def quality(self): """The quality assessment as assigned by the experimenter. Returns: str: the quality """ return self.__tuple["quality"] @property def has_nix(self): """Whether or not a nix file was recorded. Returns: bool: true or false """ return self.__tuple["has_nix"] @property def comment(self): """The comment entered by the experimenter. Returns: str: the comment """ return self.__tuple["comment"] @property def data_source(self): """Where the data is stored (at least at the time of import). Returns: str: the path to the data. """ return self.__tuple["data_source"] @property def data_host(self): """Returns the fully qualified name of the host from which the data was imported. That is, where it should be available. Returns: str: the fully qualified domain. """ return self.__tuple["data_host"] @property def setup(self): """The recording setup. Returns: str: the setup """ return self.__tuple["setup"] @property def cells(self): """The list of cells that are associated with this dataset. Returns: list of fishbook.Cell: the cells. """ cell_list = (Cells * (CellDatasetMap & self.__tuple)) return [Cell(tuple=c) for c in cell_list] @property def subjects(self): """The subjects that are associated with this dataset. Returns: list of fishbook.Subject: list of subjects """ subject_list = (Subjects * (SubjectDatasetMap & self.__tuple)) return [Subject(tuple=s) for s in subject_list] @property def repro_runs(self): """The repros that have been run in this dataset. Returns: list of fishbook.RePro: list of repro runs """ repros = (Repros * Cells * (CellDatasetMap & self.__tuple)) return [RePro(tuple=r) for r in repros] @property def samplerate(self): """Get the samplerate of the data. Returns: float: the sample rate """ return self.__samplerate @staticmethod def datasetCount(): """Returns the total number of datasets defined in the database. Returns: int: the count """ return len(Datasets()) @staticmethod def find(dataset_id=None, min_duration=None, experimenter=None, quality=None, min_date=None, max_date=None, test=False): """Find dataset entries in the database. You may restrict the search by providing the following arguments. All restrictions are connected with a logical AND. Args: dataset_id (str, optional): the id of the given Dataset, if unique, all other restrictions will be ignored. Defaults to None min_duration (float, optional): minimum duration of the recording session, if not given, any length datasets will be returned. Defaults to None. experimenter (str, optional): the name of the one who did the recording. The name does not need to be the full name. Defaults to None. quality (str, optional): the quality assigned to the dataset during recording (e.g. good, fair, poor). Defaults to None. min_date (datetime, optional): the minimum recording date. Dates may be given as datetime objects or string of the format "2010.01.01" or "2010-01-01". Defaults to None. max_date (datetime, optional): the maximum recording date. Defaults to None. test (bool, optional): defines whether this is a test run and thus whether or not the search results should be fetched, which can take a while. Defaults to False. Returns: list: list of Dataset object matching the search criteria, empty if test==True int: Count of matching results """ dataset_list = Datasets() if dataset_id: dataset_list = dataset_list & "dataset_id like '%{0:s}%'".format(dataset_id) if len(dataset_list) == 1: return [Dataset(tuple=dataset_list.fetch(as_dict=True)[0])], 1 if min_duration: dataset_list = dataset_list & "duration > %.2f" % min_duration if experimenter: dataset_list = dataset_list & "experimenter like '%{0:s}%'".format(experimenter) if quality: dataset_list = dataset_list & "quality like '{0:s}'".format(quality) if min_date and isinstance(min_date, (dt.date, str)): dataset_list = dataset_list & "recording_date >= '%s'" % min_date if max_date and isinstance(min_date, (dt.date, str)): dataset_list = dataset_list & "recording_date < '%s'" % max_date results = [] total = len(dataset_list) if not test: for i, d in enumerate(dataset_list): progress(i+1, total, "fetching %i matches" % total) results.append(Dataset(tuple=d)) return results, total def __find_samplerate(self, trace_name="V-1"): if self.has_nix and os.path.exists(os.path.join(self.data_source, self.id, '.nix')): f = nix.File.open(os.path.join(self.data_source, self.id, '.nix'), nix.FileMode.ReadOnly) b = f.blocks[0] if trace_name in b.data_arrays: trace = b.data_arrays[trace_name] if trace.dimensions[0].dimension_type == nix.DimensionType.Sample: self.__samplerate = 1./trace.dimensions[0].sampling_interval else: print("Requested trace %s has no sampled dimension!" % trace_name) else: print("Requested trace %s was not found!" % trace_name) f.close() else: stimulus_file = os.path.join(self.data_source , 'stimuli.dat') if not os.path.exists(stimulus_file): return lines = None with open(stimulus_file, 'r') as f: lines = f.readlines() for l in lines: if "sample interval1" in l: si = l.strip().split(":")[-1][:-2] break self.__samplerate = 1000. / float(si) @property def _tuple(self): return self.__tuple.copy() @property def yaml(self): settings = yaml.dump({"dataset id": self.id, "recording date": self.recording_date, "duration": self.recording_duration, "comment": self.comment, "experimenter": self.experimenter, "quality": self.quality, "data_source": self.data_source, "host": self.data_host, "setup": self.setup, "nixed": self.has_nix}) return settings def __str__(self): str = "dataset id: %s\n" % self.id str += "recorded: %s \t by: %s\n" % (self.recording_date, self.experimenter) str += "duration: %s min \t quality: %s\n" % (self.recording_duration, self.quality) str += "location: %s\t host:%s\n" % (self.data_source, self.data_host) str += "comment: %s" % self.comment return str def __repr__(self): return self.__str__() class RePro: """The RePro class represents an entry in the repro table. This is a run of a certain relacs "Research Protocol". """ def __init__(self, repro_id=None, tuple=None): if tuple: self.__tuple = tuple elif repro_id: repros = (RePro & "repro_id like '{0:s}'".format(repro_id)) results_check(repros, repro_id, "RePro ID") self.__tuple = repros.fetch(limit=1, as_dict=True)[0] else: self.__tuple = {} print("Empty RePro, not linked to any database entry!") @property def id(self): return safe_get_val(self.__tuple, "repro_id", "") @property def run(self): return safe_get_val(self.__tuple, "run", -1) @property def cell_id(self): return safe_get_val(self.__tuple, "cell_id", "") @property def cell(self): return Cell(self.cell_id) @property def dataset(self): datasets = (Cells & "cell_id = '%s'" % self.cell_id) * CellDatasetMap * Datasets d = datasets.proj('dataset_id', 'data_source', 'data_host', 'experimenter', 'setup', 'recording_date', 'quality', 'comment', 'duration', 'has_nix').fetch(limit=1, as_dict=True)[0] del d["cell_id"] return Dataset(tuple=d) @property def name(self): return safe_get_val(self.__tuple, "repro_name", "") @property def settings(self): return safe_get_val(self.__tuple, "settings", "") @property def start(self): return safe_get_val(self.__tuple, "start", 0.0) @property def duration(self): return safe_get_val(self.__tuple, "duration", 0.0) @property def stimuli(self): stims = Stimuli & "repro_id = '%s'" % self.id & "cell_id = '%s'" % self.cell_id return [Stimulus(tuple=s) for s in stims] @staticmethod def find(name=None, cell_id=None, cell_type=None, species=None, settings=[], quality=None, min_date=None, max_date=None, test=False): """ Finds Repro runs in the database. When called without arguments, all RePro runs are found and returned. Search can be narrowed by providing further search hints. **Note:** Cell type, quality, and species are ignored, if cell_id is provided **Note:** If there are many results fetching and creating objects may take a while. Consider running as test. Args: name (str, optional): The RePro name, or a part of it. Defaults to None. cell_id (str, optional): The cell_id. If given type, quality and species are ignored. Defaults to None. cell_type (str, optional): type of cell. Defaults to None. species (str, optional): The species name, or parts of it. Defaults to None. settings (list of string, optional): List of repro settings e.g. ["contrast: 20", "am: false"]. An AND connection is assumed. Defaults to None. quality (str, optional): The quality assessment. Defaults to None. min_date (datetime, optional): the minimum recording date. Dates may be given as datetime objects or string of the format "2010.01.01" or "2010-01-01". Defaults to None. max_date (datetime, optional): the maximum recording date. Defaults to None. test (bool, optional): defines whether or not the database matches should be fetched from the database. Returns: list of fishbook.RePro: list of results or empty list if test == True int: number of matches """ repros = Repros & True if name: repros = repros & "repro_name like '%{0:s}%'".format(name) if len(settings) > 0: settings_pattern = " AND ".join(["settings like '%{0:s}%'".format(s) for s in settings]) repros = repros & settings_pattern if cell_id: repros = repros & "cell_id = '{0:s}'".format(cell_id) if not cell_id and (cell_type or species or quality): cells = (Cells * CellDatasetMap * Datasets) * Subjects if cell_type: cells = cells & "cell_type like '%{0:s}%'".format(cell_type) if species: cells = cells & "species like '%{0:s}%'".format(species) if min_date and isinstance(min_date, (dt.date, str)): cells = cells & "recording_date >= '%s'" % min_date if max_date and isinstance(min_date, (dt.date, str)): cells = cells & "recording_date < '%s'" % max_date if quality: cells = cells & "quality like '{0:s}'".format(quality) p = cells.proj("quality", "experimenter") repros = repros & p results = [] total = len(repros) if not test: for i, r in enumerate(repros): results.append(RePro(tuple=r)) progress(i+1, total, "fetching %i matches" % total) return results, total @property def _tuple(self): return self.__tuple.copy() def __str__(self): str = "RePro id: %s\t repro name: %s\n" % (self.id, self.name) str += "run: %i\t on cell: %s\n" %(self.run, self.cell_id) str += "start time: %s\t duration: %s\n" % (self.start, self.duration) return str def __repr__(self): return self.__str__() @property def to_dict(self): r_settings = yaml.safe_load(self.settings.replace("\t", " ")) settings = {"repro id": self.id, "run": self.run, "cell": self.cell_id, "name": self.name, "start": self.start, "duration": self.duration, "settings":r_settings} return settings class Stimulus: """The stimulus class represents a Stimulus that was presented. A Stimulus has several properties such as the start time, the duarion """ def __init__(self, stimulus_id=None, tuple=None): if tuple: self.__tuple = tuple elif stimulus_id: stimuli = Stimuli & "stimulus_id = '%s'" %stimulus_id results_check(stimuli, stimulus_id, "Stimulus ID") self.__tuple = stimuli.fetch(limit=1, as_dict=True)[0] else: print("Empty RePro, not linked to any database entry!") def __str__(self): str = "Stimulus %s: " % safe_get_val(self.__tuple, "stimulus_id", "") str += "\nStart time/index: %0.4f/%i, duration: %.3f" % (safe_get_val(self.__tuple, "start_time", 0.0), safe_get_val(self.__tuple, "start_index", -1), safe_get_val(self.__tuple, "duration", 0.0)) return str @property def id(self): return self.__tuple["stimulus_id"] @property def name(self): return self.__tuple["stimulus_name"] @property def cell(self): embed() # FIXME return Cells & ("cell_id = %s" % self.__tuple["cell_id"]) @property def repro(self): embed() # FIXME return Repros & ("repro_id = %s" % self.__tuple["repro_id"]) @property def start_time(self): return self.__tuple["start_time"] @property def start_index(self): return self.__tuple["start_index"] @property def duration(self): return self.__tuple["duration"] @property def multi_tag_id(self): return self.__tuple["mtag_id"] @property def run(self): return self.__tuple["run"] @property def index(self): return self.__tuple["stimulus_index"] @property def settings(self): return safe_get_val(self.__tuple, "settings", "") @property def _tuple(self): return self.__tuple.copy() @staticmethod def find(cell_id=None, repro_id=None, settings=[], test=False): """Find stimulus presentations that match in certain properties. Examples: Args: cell_id (str, optional): [description]. Defaults to None. repro_id (str, optional): [description]. Defaults to None. settings (str, optional): [description]. Defaults to []]. test (bool, optional): whether or not this is a test run. When true, results will not be fetched from the database. Defaults to False. Returns: List: of matching Stimulus presentations, empty if test==True int: count of matches """ stimuli = Stimuli & True if cell_id: stimuli = stimuli & "cell_id = '{0:s}'".format(cell_id) if repro_id: stimuli = stimuli & "repro_id = '{0:s}'".format(repro_id) if len(settings) > 0: settings_pattern = " AND ".join(["settings like '%{0:s}%'".format(s) for s in settings]) stimuli = stimuli & settings_pattern results = [] total = len(stimuli) if not test: for i, s in enumerate(stimuli): results.append(Stimulus(tuple=s)) progress(i+1, total, "fetching %i matches" % total) return results, total def __repr__(self): return self.__str__() @property def to_dict(self): s_settings = yaml.safe_load(self.settings.replace("\t", " ")) settings = {"id": self.id, "run": self.run, "stimulus name" : self.name, "stimulus index": self.index, "duration": self.duration, "start time": self.start_time, "name": self.name, "settings": s_settings} return settings class Subject: """Representation of the recorded subject's properties. A subject is defined by the its *id*, and its *species*. The """ def __init__(self, subject_id=None, tuple=None): if tuple: self.__tuple = tuple elif subject_id: subjects = Subjects & "subject_id like '{0:s}'".format(subject_id) results_check(subjects, subject_id, "Subject ID") self.__tuple = subjects.fetch()[0] else: self.__tuple = {} print("Empty Subject, not linked to any database entry!") @property def id(self): return self.__tuple["subject_id"] @property def species(self): return self.__tuple["species"] @property def cells(self): cs = Cells & self.__tuple return [Cell(tuple=c) for c in cs] @property def properties(self): return (SubjectProperties & self.__tuple).fetch(as_dict=True) @property def _tuple(self): return self.__tuple.copy() @staticmethod def find(species=None, test=False): """Find all subjects matching the given species. Args: species ([type], optional): [description]. Defaults to None. test (bool, optional): [description]. Defaults to False. Returns: [type]: [description] """ subjs = Subjects & True if species: subjs = (Subjects & "species like '%{0:s}%'".format(species)) results = [] total = len(subjs) if not test: for i, s in enumerate(subjs): results.append(Subject(tuple=s)) progress(i+1, total, "fetching %i matches" % total) return results, total @staticmethod def unique_species(): """Unique list of species names. Returns: np.array: unique names """ all_species = (Subjects & True).fetch("species") return np.unique(all_species) def __str__(self): str = "Subject: %s\n" % self.id str += "Species: %s\n" % self.species return str if __name__ == "__main__": from IPython import embed cell = Cell("2010-04-16-ak") embed()