<commit_before># -*- coding: utf-8 -*-
import acq4.util.DataManager as DataManager
import acq4.util.SequenceRunner as SequenceRunner
from collections import OrderedDict
import functools
from acq4.util.metaarray import *
import numpy as np

protocolNames = {
    'IV Curve': ('cciv.*', 'vciv.*'),
    'Photostim Scan': (),
    'Photostim Power Series': (),
}

# note: make sure the names, if single, are followed by ',', so as to enforce elements of tuple
deviceNames = {
    'Clamp': ('Clamp1', 'Clamp2', 'AxoPatch200', 'AxoProbe', 'MultiClamp1', 'MultiClamp2'),
    'Camera': ('Camera',),
    'Laser': ('Laser-UV', 'Laser-Blue', 'Laser-2P'),
    'LED-Blue': ('LED-Blue',),
}

# current and voltage clamp modes that are know to us
ic_modes = ['IC', 'CC', 'IClamp', 'ic', 'I-Clamp Fast', 'I-Clamp Slow']
vc_modes = ['VC', 'VClamp', 'vc']  # list of VC modes


"""Function library for formalizing the raw data structures used in analysis.
This provides a layer of abstraction between the raw data and the analysis routines.

Should allow simple questions like
    Were any scan protocols run under this cell?
    Is this a sequence protocol or a single?
    Give me a meta-array linking all of the data in a sequence (will have to use hdf5 for this?)
    Give me a meta-array of all images from 'Camera' in a sequence (will have to use hdf5 for this?)
    
    Give me the clamp device for this protocol run
    tell me the temperature of this run
    tell me the holding potential for this clamp data
    possibly DB integration?
    
    When did the laser stimulation occur, for how long, and at what power level?
    
Notes:
    Should be able to easily switch to a different data model
    
"""

def knownClampNames():
    return deviceNames['Clamp']

def isSequence(dh):
    """Return true if dh is a directory handle for a protocol sequence."""
    return dirType(dh) == 'ProtocolSequence'
    #if 'sequenceParams' in dh.info():
        #return True
    #else:
        #return False
    
def dirType(dh, allowRecurse=True):
    """
    Return a string representing the type of data stored in a directory.
    Usually, this is provided in the meta-info for the directory, but in a few
    cases (old data formats) we need to do a little more probing.
    Returns None if the type cannot be determined.
    """
    info = dh.info()
    type = info.get('dirType', None)
    if type is None:
        if '__object_type__' in info:
            type = info['__object_type__']
        elif dh.name()[-5:] == 'Patch':
            type = 'Patch'
        elif 'protocol' in info:
            if 'sequenceParams' in info:
                type = 'ProtocolSequence'  
            else:
                type = 'Protocol'  ## an individual protocol run, NOT a single run from within a sequence
        
        else:
            try:
                assert allowRecurse
                parent = dh.parent()
                if dirType(parent, allowRecurse=False) == 'ProtocolSequence':
                    type = 'Protocol'
                #else:
                    #raise Exception()
            except:
                pass
                #raise Exception("Can't determine type for dir %s" % dh.name())
    return type

def listSequenceParams(dh):
    """Given a directory handle for a protocol sequence, return the dict of sequence parameters"""
    try:
        return dh.info()['sequenceParams']
    except KeyError:
        raise Exception("Directory '%s' does not appear to be a protocol sequence." % dh.name())

## what's this for?
#def listWaveGenerator(dh):
#    try:
#        return dh.info()['waveGeneratorWidget']
#    except KeyError:
#        raise Exception("Directory '%s' does not appear to be have a wave Generator." % dh.name())

def buildSequenceArray(*args, **kargs):
    """Builds a MetaArray of data compiled across a sequence. 
    Arguments:
        dh:      directory handle for the protocol sequence
        func:    a function (optional) that returns an array or scalar, given a protocol dir handle.
                 If func is None, return an object array containing the DirHandles (forces join=False)
        join:    If True (default), attempt to join all results into a single array. This assumes that
                 func returns arrays of equal shape for every point in the parameter space.
                 If False, just return an object array pointing to individual results.
        truncate: If join=True and some elements differ in shape, truncate to the smallest shape
        fill:    If join=True, pre-fill the empty array with this value. Any points in the
                 parameter space with no data will be left with this value.
        
    Example: Return an array of all primary-channel clamp recordings across a sequence 
        buildSequenceArray(seqDir, lambda protoDir: getClampFile(protoDir).read()['primary'])"""
        
    for i,m in buildSequenceArrayIter(*args, **kargs):
        if m is None:
            return i
        
def buildSequenceArrayIter(dh, func=None, join=True, truncate=False, fill=None):
    """Iterator for buildSequenceArray that yields progress updates."""
        
    if func is None:
        func = lambda dh: dh
        join = False
        
    params = listSequenceParams(dh)
    #inds = OrderedDict([(k, range(len(v))) for k,v in params.iteritems()])
    #def runFunc(dh, func, params):
        #name = '_'.join(['%03d'%n for n in params.values()])
        #fh = dh[name]
        #return func(fh)
    #data = SequenceRunner.runSequence(functools.partial(runFunc, dh, func), inds, inds.keys())
    subDirs = dh.subDirs()
    if len(subDirs) == 0:
        yield None, None
    
    ## set up meta-info for sequence axes
    seqShape = tuple([len(p) for p in params.itervalues()])
    info = [[] for i in range(len(seqShape))]
    i = 0
    for k,v in params.iteritems():
        info[i] = {'name': k, 'values': np.array(v)}
        i += 1
    
    ## get a data sample
    first = func(dh[subDirs[0]])
    
    ## build empty MetaArray
    if join:
        shape = seqShape + first.shape
        if isinstance(first, MetaArray):
            info = info + first._info
        else:
            info = info + [[] for i in range(first.ndim+1)]
        data = MetaArray(np.empty(shape, first.dtype), info=info)
        if fill is not None:
            data[:] = fill
        
    else:
        shape = seqShape
        info = info + []
        data = MetaArray(np.empty(shape, object), info=info)

    ## fill data
    i = 0
    if join and truncate:
        minShape = first.shape
        for name in subDirs:
            subd = dh[name]
            d = func(subd)<commit_msg>minor datamodel fix<commit_after># -*- coding: utf-8 -*-
import acq4.util.DataManager as DataManager
import acq4.util.SequenceRunner as SequenceRunner
from collections import OrderedDict
import functools
from acq4.util.metaarray import *
import numpy as np

protocolNames = {
    'IV Curve': ('cciv.*', 'vciv.*'),
    'Photostim Scan': (),
    'Photostim Power Series': (),
}

# note: make sure the names, if single, are followed by ',', so as to enforce elements of tuple
deviceNames = {
    'Clamp': ('Clamp1', 'Clamp2', 'AxoPatch200', 'AxoProbe', 'MultiClamp1', 'MultiClamp2'),
    'Camera': ('Camera',),
    'Laser': ('Laser-UV', 'Laser-Blue', 'Laser-2P'),
    'LED-Blue': ('LED-Blue',),
}

# current and voltage clamp modes that are know to us
ic_modes = ['IC', 'CC', 'IClamp', 'ic', 'I-Clamp Fast', 'I-Clamp Slow']
vc_modes = ['VC', 'VClamp', 'vc']  # list of VC modes


"""Function library for formalizing the raw data structures used in analysis.
This provides a layer of abstraction between the raw data and the analysis routines.

Should allow simple questions like
    Were any scan protocols run under this cell?
    Is this a sequence protocol or a single?
    Give me a meta-array linking all of the data in a sequence (will have to use hdf5 for this?)
    Give me a meta-array of all images from 'Camera' in a sequence (will have to use hdf5 for this?)
    
    Give me the clamp device for this protocol run
    tell me the temperature of this run
    tell me the holding potential for this clamp data
    possibly DB integration?
    
    When did the laser stimulation occur, for how long, and at what power level?
    
Notes:
    Should be able to easily switch to a different data model
    
"""

def knownClampNames():
    return deviceNames['Clamp']

def isSequence(dh):
    """Return true if dh is a directory handle for a protocol sequence."""
    return dirType(dh) == 'ProtocolSequence'
    #if 'sequenceParams' in dh.info():
        #return True
    #else:
        #return False
    
def dirType(dh, allowRecurse=True):
    """
    Return a string representing the type of data stored in a directory.
    Usually, this is provided in the meta-info for the directory, but in a few
    cases (old data formats) we need to do a little more probing.
    Returns None if the type cannot be determined.
    """
    info = dh.info()
    type = info.get('dirType', None)
    if type is None:
        if '__object_type__' in info:
            type = info['__object_type__']
        elif dh.name()[-5:] == 'Patch':
            type = 'Patch'
        elif 'protocol' in info:
            if 'sequenceParams' in info:
                type = 'ProtocolSequence'  
            else:
                type = 'Protocol'  ## an individual protocol run, NOT a single run from within a sequence
        
        else:
            try:
                assert allowRecurse
                parent = dh.parent()
                if dirType(parent, allowRecurse=False) == 'ProtocolSequence':
                    type = 'Protocol'
                #else:
                    #raise Exception()
            except:
                pass
                #raise Exception("Can't determine type for dir %s" % dh.name())
    return type

def listSequenceParams(dh):
    """Given a directory handle for a protocol sequence, return the dict of sequence parameters"""
    try:
        return dh.info()['sequenceParams']
    except KeyError:
        raise Exception("Directory '%s' does not appear to be a protocol sequence." % dh.name())

## what's this for?
#def listWaveGenerator(dh):
#    try:
#        return dh.info()['waveGeneratorWidget']
#    except KeyError:
#        raise Exception("Directory '%s' does not appear to be have a wave Generator." % dh.name())

def buildSequenceArray(*args, **kargs):
    """Builds a MetaArray of data compiled across a sequence. 
    Arguments:
        dh:      directory handle for the protocol sequence
        func:    a function (optional) that returns an array or scalar, given a protocol dir handle.
                 If func is None, return an object array containing the DirHandles (forces join=False)
        join:    If True (default), attempt to join all results into a single array. This assumes that
                 func returns arrays of equal shape for every point in the parameter space.
                 If False, just return an object array pointing to individual results.
        truncate: If join=True and some elements differ in shape, truncate to the smallest shape
        fill:    If join=True, pre-fill the empty array with this value. Any points in the
                 parameter space with no data will be left with this value.
        
    Example: Return an array of all primary-channel clamp recordings across a sequence 
        buildSequenceArray(seqDir, lambda protoDir: getClampFile(protoDir).read()['primary'])"""
        
    for i,m in buildSequenceArrayIter(*args, **kargs):
        if m is None:
            return i
        
def buildSequenceArrayIter(dh, func=None, join=True, truncate=False, fill=None):
    """Iterator for buildSequenceArray that yields progress updates."""
        
    if func is None:
        func = lambda dh: dh
        join = False
        
    params = listSequenceParams(dh)
    #inds = OrderedDict([(k, range(len(v))) for k,v in params.iteritems()])
    #def runFunc(dh, func, params):
        #name = '_'.join(['%03d'%n for n in params.values()])
        #fh = dh[name]
        #return func(fh)
    #data = SequenceRunner.runSequence(functools.partial(runFunc, dh, func), inds, inds.keys())
    subDirs = dh.subDirs()
    if len(subDirs) == 0:
        yield None, None
    
    ## set up meta-info for sequence axes
    seqShape = tuple([len(p) for p in params.itervalues()])
    info = [[] for i in range(len(seqShape))]
    i = 0
    for k,v in params.iteritems():
        info[i] = {'name': k, 'values': np.array(v)}
        i += 1
    
    ## get a data sample
    first = func(dh[subDirs[0]])
    
    ## build empty MetaArray
    if join:
        shape = seqShape + first.shape
        if isinstance(first, MetaArray):
            info = info + first._info
        else:
            info = info + [{} for i in range(first.ndim+1)]
        data = MetaArray(np.empty(shape, first.dtype), info=info)
        if fill is not None:
            data[:] = fill
        
    else:
        shape = seqShape
        info = info + []
        data = MetaArray(np.empty(shape, object), info=info)

    ## fill data
    i = 0
    if join and truncate:
        minShape = first.shape
        for name in subDirs:
            subd = dh[name]
            d = func(subd)
{'fraction_non_alphanumeric': 0.10007513148009016, 'fraction_numerical': 0.0034560480841472575, 'mean_word_length': 2.436870642912471, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4257345', 'n_tokens_mistral': 3835, 'n_tokens_neox': 3683, 'n_words': 1499}