<commit_before># encoding: utf-8
"""
Classe for reading data in CED spike2 files (.smr).

This code is based on:
 - sonpy, written by <NAME> <<EMAIL>>
    Disponible here ::
    http://www.neuro.ki.se/broberger/

and sonpy come from :
 - SON Library 2.0 for MATLAB, written by <NAME> at
    King's College London. See http://www.kcl.ac.uk/depsta/biomedical/cfnr/lidierth.html

This IO support old (<v6) and new files (>v7) of spike2


Depend on: 

Supported : Read

Author: sgarcia

"""

from .baseio import BaseIO
from ..core import *
from .tools import create_many_to_one_relationship

import numpy as np
from numpy import dtype, zeros, fromstring, empty
import quantities as pq

import os, sys

PY3K = (sys.version_info[0] == 3)


class Spike2IO(BaseIO):
    """
    Class for reading data from CED spike2.
    
    Usage:
        >>> from neo import io
        >>> r = io.Spike2IO( filename = 'File_spike2_1.smr')
        >>> seg = r.read_segment(lazy = False, cascade = True,)
        >>> print seg.analogsignals
        >>> print seg.spiketrains
        >>> print seg.eventarrays
    
    """
    
    is_readable        = True
    is_writable        = False
    
    supported_objects  = [ Segment , AnalogSignal , EventArray, SpikeTrain]
    readable_objects   = [Segment]
    writeable_objects  = [ ]      


    has_header         = False
    is_streameable     = False
    read_params        = {   Segment : [ ], }
    write_params       = None

    name               = 'Spike 2 CED'
    extensions          = [ 'smr' ]
    
    mode = 'file'
    
    def __init__(self , filename = None) :
        """
        This class read a smr file.
        
        Arguments:
            filename : the filename 
        """
        BaseIO.__init__(self)
        self.filename = filename

    def read_segment(self ,
                                            lazy = False,
                                            cascade = True,

                                                ):
        """
        Arguments:
        """
        
        
        header = self.read_header(filename = self.filename)
        
        #~ print header
        fid = open(self.filename, 'rb')
        
        seg  = Segment(
                                    file_origin = os.path.basename(self.filename),
                                    ced_version = str(header.system_id),
                                    )
        
        if not cascade:
            return seg
        
        def addannotations(ob, channelHeader):
            ob.annotate(title = channelHeader.title)
            ob.annotate(physical_channel_index = channelHeader.phy_chan)
            ob.annotate(comment = channelHeader.comment)
        
        for i in range(header.channels) :
            channelHeader = header.channelHeaders[i]
            
            #~ print 'channel' , i , 'kind' ,  channelHeader.kind
            
            if channelHeader.kind !=0:
                #~ print '####'
                #~ print 'channel' , i, 'kind' , channelHeader.kind , channelHeader.type , channelHeader.phy_chan
                #~ print channelHeader
                pass
            
            if channelHeader.kind in [1, 9]:
                #~ print 'analogChanel'
                anaSigs = self.readOneChannelContinuous( fid, i, header ,lazy = lazy)
                #~ print 'nb sigs', len(anaSigs) , ' sizes : ',
                for anaSig in anaSigs :
                    addannotations(anaSig, channelHeader)
                    seg.analogsignals.append( anaSig )
                    #~ print sig.signal.size,
                #~ print ''
                    
            elif channelHeader.kind in  [2, 3, 4, 5, 8] :
                ea = self.readOneChannelEventOrSpike( fid, i, header , lazy = lazy)
                addannotations(ea, channelHeader)
                seg.eventarrays.append(ea)
                
            elif channelHeader.kind in  [6,7] :
                sptr = self.readOneChannelEventOrSpike( fid, i, header, lazy = lazy )
                if sptr is not None:
                    addannotations(sptr, channelHeader)
                    seg.spiketrains.append(sptr)
            
        fid.close()
        
        create_many_to_one_relationship(seg)
        return seg
        
        
    def read_header(self , filename = ''):
        
        fid = open(filename, 'rb')
        header = HeaderReader(fid,   dtype(headerDescription))
        #~ print 'chan_size' , header.chan_size
        
        
        if header.system_id < 6:
            header.dtime_base = 1e-6
            header.datetime_detail = 0
            header.datetime_year = 0
        
        channelHeaders = [ ]
        for i in range(header.channels):
            # read global channel header
            fid.seek(512 + 140*i) # TODO verifier i ou i-1
            channelHeader = HeaderReader(fid, dtype(channelHeaderDesciption1))
            if channelHeader.kind in [1, 6]:
                dt = [('scale' , 'f4'),
                      ('offset' , 'f4'),
                      ('unit' , 'S6'),]
                channelHeader += HeaderReader(fid, dtype(dt))
                if header.system_id < 6:
                    channelHeader += HeaderReader(fid, dtype([ ('divide' , 'i4')]) )#i8
                else : 
                    channelHeader +=HeaderReader(fid, dtype([ ('interleave' , 'i4')]) )#i8
            if channelHeader.kind in [7, 9]:
                dt = [('min' , 'f4'),
                      ('max' , 'f4'),
                      ('unit' , 'S6'),]
                channelHeader += HeaderReader(fid, dtype(dt))
                if header.system_id < 6:
                    channelHeader += HeaderReader(fid, dtype([ ('divide' , 'i4')]))#i8
                else :
                    channelHeader += HeaderReader(fid, dtype([ ('interleave' , 'i4')]) )#i8
            if channelHeader.kind in [4]:
                dt = [('init_low' , 'u1'),
                      ('next_low' , 'u1'),]
                channelHeader += HeaderReader(fid, dtype(dt))
            
            channelHeader.type = dict_kind[channelHeader.kind]
            channelHeaders.append(channelHeader)
            
        header.channelHeaders = channelHeaders
        
        fid.close()
        return header

    
    def readOneChannelContinuous(self , fid, channel_num, header ,lazy = True):
        # read AnalogSignal
        channelHeader = header.channelHeaders[channel_num]
        
        
        # data type
        if channelHeader.kind == 1:
            dt = np.dtype('i2')
        elif channelHeader.kind == 9:
            dt = np.dtype('f4')
        
        # sample rate
        if header.system_id in [1,2,3,4,5]: # Before version 5
            sample_interval = (channelHeader.divide*header.us_per_time*header.time_per_adc)*1e-6
        else :
            sample_interval = (channelHeader.l_chan_dvd*header.us_per_time*header.dtime_base)
        sampling_rate = (1./sample_interval)*pq.Hz
        
        # read blocks header to preallocate memory by jumping block to block
        fid.seek(channelHeader.firstblock)
        blocksize = [ 0 ]
        starttimes = [ ]
        for b in range(channelHeader.blocks) :
            blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
            if len(blocksize) > len(starttimes):
                starttimes.append(blockHeader.start_time)
            blocksize[-1] += blockHeader.items
            
            if blockHeader.succ_block > 0 :
                # this is ugly but CED do not garanty continuity in AnalogSignal
                fid.seek(blockHeader.succ_block)
                nextBlockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
                sample_interval = (blockHeader.end_time-blockHeader.start_time)/(blockHeader.items-1)
                interval_with_next = nextBlockHeader.start_time - blockHeader.end_time
                if interval_with_next > sample_interval:
                    blocksize.append(0)
                fid.seek(blockHeader.succ_block)
        
        anaSigs = [ ]
        if channelHeader.unit in unit_convert:
            unit = pq.Quantity(1, unit_convert[channelHeader.unit] )
        else:
            #print channelHeader.unit
            try:
                unit = pq.Quantity(1, channelHeader.unit )
            except:
                unit = pq.Quantity(1, '')
            
        for b,bs in enumerate(blocksize ):
            if lazy:
                signal = [ ]*unit
            else:
                signal = empty( bs , dtype = 'f4') * unit
            anaSig = AnalogSignal(signal ,
                                                            sampling_rate = sampling_rate,
                                                            t_start = starttimes[b]*header.us_per_time * header.dtime_base * pq.s,
                                                            )
            anaSig.annotate(channel_index = channel_num)
            anaSigs.append( anaSig )
        
        if  lazy:
            for s, anaSig in enumerate(anaSigs):
                anaSig.lazy_shape = blocksize[s]
            
        else:
            # read data  by jumping block to block
            fid.seek(channelHeader.firstblock)
            pos = 0
            numblock = 0
            for b in range(channelHeader.blocks) :
                blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
                # read data
                sig = fromstring( fid.read(blockHeader.items*dt.itemsize) , dtype = dt)
                anaSigs[numblock][pos:pos+sig.size] = sig.astype('f4')*unit
                pos += sig.size
                if pos >= blocksize[numblock] :
                    numblock += 1
                    pos = 0
                # jump to next block
                if blockHeader.succ_block > 0 :
                    fid.seek(blockHeader.succ_block)
            
        # convert for int16
        if dt.kind == 'i' :
            for anaSig in anaSigs :
                anaSig *= channelHeader.scale/ 6553.6
                anaSig += channelHeader.offset*unit
        
        return anaSigs
    
    
    def readOneChannelEventOrSpike(self , fid, channel_num, header ,lazy = True):
        # return SPikeTrain or EventArray
        channelHeader = header.channelHeaders[channel_num]
        if channelHeader.firstblock <0: return
        if channelHeader.kind not in [2, 3, 4 , 5 , 6 ,7, 8]: return
        
        ## Step 1 : type of blocks
        if channelHeader.kind in [2, 3, 4]:
            # Event data
            format = [('tick' , 'i4') ]
        elif channelHeader.kind in [5]:
            # Marker data
            format = [('tick' , 'i4') , ('marker' , 'i4') ]
        elif channelHeader.kind in [6]:
            # AdcMark data
            format = [('tick' , 'i4') , ('marker' , 'i4')  , ('adc' , 'S%d' %channelHeader.n_extra   )]
        elif channelHeader.kind in [7]:
            #  RealMark data
            format = [('tick' , 'i4') , ('marker' , 'i4')  , ('real' , 'S%d' %channelHeader.n_extra   )]
        elif channelHeader.kind in [8]:
            # TextMark data
            format = [('tick' , 'i4') , ('marker' , 'i4')  ,  ('label' , 'S%d'%channelHeader.n_extra)]
        dt = dtype(format)
        
            
        ## Step 2 : first read for allocating mem
        fid.seek(channelHeader.firstblock)
        totalitems = 0
        for b in range(channelHeader.blocks) :
            blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
            totalitems += blockHeader.items
            if blockHeader.succ_block > 0 :
                fid.seek(blockHeader.succ_block)
        #~ print 'totalitems' , totalitems
        
        if lazy :
            if channelHeader.kind in [2, 3, 4 , 5 , 8]:
                ea = EventArray(  )
                ea.annotate(channel_index = channel_num)
                ea.lazy_shape = totalitems
                return ea
                
            elif channelHeader.kind in [6 ,7]:
                sptr = SpikeTrain([ ]*pq.s, t_stop=1e99)  # correct value for t_stop to be put in later
                sptr.annotate(channel_index = channel_num)
                sptr.lazy_shape = totalitems
                return sptr
        
        else:
            alltrigs = zeros( totalitems , dtype = dt)
            ## Step 3 : read
            fid.seek(channelHeader.firstblock)
            pos = 0
            for b in range(channelHeader.blocks) :
                blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
                # read all events in block
                trigs = fromstring( fid.read( blockHeader.items*dt.itemsize)  , dtype = dt)
                
                alltrigs[pos:pos+trigs.size] = trigs
                pos += trigs.size
                if blockHeader.succ_block > 0 :
                    fid.seek(blockHeader.succ_block)
            
            ## Step 3 convert in neo standard class : eventarrays or spiketrains
            alltimes = alltrigs['tick'].astype('f')*header.us_per_time * header.dtime_base*pq.s
            
            if channelHeader.kind in [2, 3, 4 , 5 , 8]:
                #events
                ea = EventArray(  )
                ea.annotate(channel_index = channel_num)
                ea.times = alltimes
                if channelHeader.kind >= 5:
                    # Spike2 marker is closer to label sens of neo
                    ea.labels = alltrigs['marker'].astype('S')
                if channelHeader.kind == 8:
                    ea.annotate(extra_labels = alltrigs['label'])<commit_msg>Patch from <EMAIL><commit_after># encoding: utf-8
"""
Classe for reading data in CED spike2 files (.smr).

This code is based on:
 - sonpy, written by <NAME> <<EMAIL>>
    Disponible here ::
    http://www.neuro.ki.se/broberger/

and sonpy come from :
 - SON Library 2.0 for MATLAB, written by <NAME> at
    King's College London. See http://www.kcl.ac.uk/depsta/biomedical/cfnr/lidierth.html

This IO support old (<v6) and new files (>v7) of spike2


Depend on: 

Supported : Read

Author: sgarcia

"""

from .baseio import BaseIO
from ..core import *
from .tools import create_many_to_one_relationship

import numpy as np
from numpy import dtype, zeros, fromstring, empty
import quantities as pq

import os, sys

PY3K = (sys.version_info[0] == 3)


class Spike2IO(BaseIO):
    """
    Class for reading data from CED spike2.
    
    Usage:
        >>> from neo import io
        >>> r = io.Spike2IO( filename = 'File_spike2_1.smr')
        >>> seg = r.read_segment(lazy = False, cascade = True,)
        >>> print seg.analogsignals
        >>> print seg.spiketrains
        >>> print seg.eventarrays
    
    """
    
    is_readable        = True
    is_writable        = False
    
    supported_objects  = [ Segment , AnalogSignal , EventArray, SpikeTrain]
    readable_objects   = [Segment]
    writeable_objects  = [ ]      


    has_header         = False
    is_streameable     = False
    read_params        = {   Segment : [ ], }
    write_params       = None

    name               = 'Spike 2 CED'
    extensions          = [ 'smr' ]
    
    mode = 'file'
    
    def __init__(self , filename = None) :
        """
        This class read a smr file.
        
        Arguments:
            filename : the filename 
        """
        BaseIO.__init__(self)
        self.filename = filename

    def read_segment(self ,
                                            lazy = False,
                                            cascade = True,

                                                ):
        """
        Arguments:
        """
        
        
        header = self.read_header(filename = self.filename)
        
        #~ print header
        fid = open(self.filename, 'rb')
        
        seg  = Segment(
                                    file_origin = os.path.basename(self.filename),
                                    ced_version = str(header.system_id),
                                    )
        
        if not cascade:
            return seg
        
        def addannotations(ob, channelHeader):
            ob.annotate(title = channelHeader.title)
            ob.annotate(physical_channel_index = channelHeader.phy_chan)
            ob.annotate(comment = channelHeader.comment)
        
        for i in range(header.channels) :
            channelHeader = header.channelHeaders[i]
            
            #~ print 'channel' , i , 'kind' ,  channelHeader.kind
            
            if channelHeader.kind !=0:
                #~ print '####'
                #~ print 'channel' , i, 'kind' , channelHeader.kind , channelHeader.type , channelHeader.phy_chan
                #~ print channelHeader
                pass
            
            if channelHeader.kind in [1, 9]:
                #~ print 'analogChanel'
                anaSigs = self.readOneChannelContinuous( fid, i, header ,lazy = lazy)
                #~ print 'nb sigs', len(anaSigs) , ' sizes : ',
                for anaSig in anaSigs :
                    addannotations(anaSig, channelHeader)
                    seg.analogsignals.append( anaSig )
                    #~ print sig.signal.size,
                #~ print ''
                    
            elif channelHeader.kind in  [2, 3, 4, 5, 8] :
                ea = self.readOneChannelEventOrSpike( fid, i, header , lazy = lazy)
                addannotations(ea, channelHeader)
                seg.eventarrays.append(ea)
                
            elif channelHeader.kind in  [6,7] :
                sptr = self.readOneChannelEventOrSpike( fid, i, header, lazy = lazy )
                if sptr is not None:
                    addannotations(sptr, channelHeader)
                    seg.spiketrains.append(sptr)
            
        fid.close()
        
        create_many_to_one_relationship(seg)
        return seg
        
        
    def read_header(self , filename = ''):
        
        fid = open(filename, 'rb')
        header = HeaderReader(fid,   dtype(headerDescription))
        #~ print 'chan_size' , header.chan_size
        
        
        if header.system_id < 6:
            header.dtime_base = 1e-6
            header.datetime_detail = 0
            header.datetime_year = 0
        
        channelHeaders = [ ]
        for i in range(header.channels):
            # read global channel header
            fid.seek(512 + 140*i) # TODO verifier i ou i-1
            channelHeader = HeaderReader(fid, dtype(channelHeaderDesciption1))
            if channelHeader.kind in [1, 6]:
                dt = [('scale' , 'f4'),
                      ('offset' , 'f4'),
                      ('unit' , 'S6'),]
                channelHeader += HeaderReader(fid, dtype(dt))
                if header.system_id < 6:
                    channelHeader += HeaderReader(fid, dtype([ ('divide' , 'i4')]) )#i8
                else : 
                    channelHeader +=HeaderReader(fid, dtype([ ('interleave' , 'i4')]) )#i8
            if channelHeader.kind in [7, 9]:
                dt = [('min' , 'f4'),
                      ('max' , 'f4'),
                      ('unit' , 'S6'),]
                channelHeader += HeaderReader(fid, dtype(dt))
                if header.system_id < 6:
                    channelHeader += HeaderReader(fid, dtype([ ('divide' , 'i4')]))#i8
                else :
                    channelHeader += HeaderReader(fid, dtype([ ('interleave' , 'i4')]) )#i8
            if channelHeader.kind in [4]:
                dt = [('init_low' , 'u1'),
                      ('next_low' , 'u1'),]
                channelHeader += HeaderReader(fid, dtype(dt))
            
            channelHeader.type = dict_kind[channelHeader.kind]
            channelHeaders.append(channelHeader)
            
        header.channelHeaders = channelHeaders
        
        fid.close()
        return header

    
    def readOneChannelContinuous(self , fid, channel_num, header ,lazy = True):
        # read AnalogSignal
        channelHeader = header.channelHeaders[channel_num]
        
        
        # data type
        if channelHeader.kind == 1:
            dt = np.dtype('i2')
        elif channelHeader.kind == 9:
            dt = np.dtype('f4')
        
        # sample rate
        if header.system_id in [1,2,3,4,5]: # Before version 5
            sample_interval = (channelHeader.divide*header.us_per_time*header.time_per_adc)*1e-6
        else :
            sample_interval = (channelHeader.l_chan_dvd*header.us_per_time*header.dtime_base)
        sampling_rate = (1./sample_interval)*pq.Hz
        
        # read blocks header to preallocate memory by jumping block to block
        fid.seek(channelHeader.firstblock)
        blocksize = [ 0 ]
        starttimes = [ ]
        for b in range(channelHeader.blocks) :
            blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
            if len(blocksize) > len(starttimes):
                starttimes.append(blockHeader.start_time)
            blocksize[-1] += blockHeader.items
            
            if blockHeader.succ_block > 0 :
                # this is ugly but CED do not garanty continuity in AnalogSignal
                fid.seek(blockHeader.succ_block)
                nextBlockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
                sample_interval = (blockHeader.end_time-blockHeader.start_time)/(blockHeader.items-1)
                interval_with_next = nextBlockHeader.start_time - blockHeader.end_time
                if interval_with_next > sample_interval:
                    blocksize.append(0)
                fid.seek(blockHeader.succ_block)
        
        anaSigs = [ ]
        if channelHeader.unit in unit_convert:
            unit = pq.Quantity(1, unit_convert[channelHeader.unit] )
        else:
            #print channelHeader.unit
            try:
                unit = pq.Quantity(1, channelHeader.unit )
            except:
                unit = pq.Quantity(1, '')
            
        for b,bs in enumerate(blocksize ):
            if lazy:
                signal = [ ]*unit
            else:
                signal = empty( bs , dtype = 'f4') * unit
            anaSig = AnalogSignal(signal ,
                                                            sampling_rate = sampling_rate,
                                                            t_start = starttimes[b]*header.us_per_time * header.dtime_base * pq.s,
                                                            )
            anaSig.annotate(channel_index = channel_num)
            anaSigs.append( anaSig )
        
        if  lazy:
            for s, anaSig in enumerate(anaSigs):
                anaSig.lazy_shape = blocksize[s]
            
        else:
            # read data  by jumping block to block
            fid.seek(channelHeader.firstblock)
            pos = 0
            numblock = 0
            for b in range(channelHeader.blocks) :
                blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
                # read data
                sig = fromstring( fid.read(blockHeader.items*dt.itemsize) , dtype = dt)
                anaSigs[numblock][pos:pos+sig.size] = sig.astype('f4')*unit
                pos += sig.size
                if pos >= blocksize[numblock] :
                    numblock += 1
                    pos = 0
                # jump to next block
                if blockHeader.succ_block > 0 :
                    fid.seek(blockHeader.succ_block)
            
        # convert for int16
        if dt.kind == 'i' :
            for anaSig in anaSigs :
                anaSig *= channelHeader.scale/ 6553.6
                anaSig += channelHeader.offset*unit
        
        return anaSigs
    
    
    def readOneChannelEventOrSpike(self , fid, channel_num, header ,lazy = True):
        # return SPikeTrain or EventArray
        channelHeader = header.channelHeaders[channel_num]
        if channelHeader.firstblock <0: return
        if channelHeader.kind not in [2, 3, 4 , 5 , 6 ,7, 8]: return
        
        ## Step 1 : type of blocks
        if channelHeader.kind in [2, 3, 4]:
            # Event data
            format = [('tick' , 'i4') ]
        elif channelHeader.kind in [5]:
            # Marker data
            format = [('tick' , 'i4') , ('marker' , 'i4') ]
        elif channelHeader.kind in [6]:
            # AdcMark data
            format = [('tick' , 'i4') , ('marker' , 'i4')  , ('adc' , 'S%d' %channelHeader.n_extra   )]
        elif channelHeader.kind in [7]:
            #  RealMark data
            format = [('tick' , 'i4') , ('marker' , 'i4')  , ('real' , 'S%d' %channelHeader.n_extra   )]
        elif channelHeader.kind in [8]:
            # TextMark data
            format = [('tick' , 'i4') , ('marker' , 'i4')  ,  ('label' , 'S%d'%channelHeader.n_extra)]
        dt = dtype(format)
        
            
        ## Step 2 : first read for allocating mem
        fid.seek(channelHeader.firstblock)
        totalitems = 0
        for b in range(channelHeader.blocks) :
            blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
            totalitems += blockHeader.items
            if blockHeader.succ_block > 0 :
                fid.seek(blockHeader.succ_block)
        #~ print 'totalitems' , totalitems
        
        if lazy :
            if channelHeader.kind in [2, 3, 4 , 5 , 8]:
                ea = EventArray(  )
                ea.annotate(channel_index = channel_num)
                ea.lazy_shape = totalitems
                return ea
                
            elif channelHeader.kind in [6 ,7]:
                sptr = SpikeTrain([ ]*pq.s, t_stop=1e99)  # correct value for t_stop to be put in later
                sptr.annotate(channel_index = channel_num)
                sptr.lazy_shape = totalitems
                return sptr
        
        else:
            alltrigs = zeros( totalitems , dtype = dt)
            ## Step 3 : read
            fid.seek(channelHeader.firstblock)
            pos = 0
            for b in range(channelHeader.blocks) :
                blockHeader = HeaderReader(fid, dtype(blockHeaderDesciption))
                # read all events in block
                trigs = fromstring( fid.read( blockHeader.items*dt.itemsize)  , dtype = dt)
                
                alltrigs[pos:pos+trigs.size] = trigs
                pos += trigs.size
                if blockHeader.succ_block > 0 :
                    fid.seek(blockHeader.succ_block)
            
            ## Step 3 convert in neo standard class : eventarrays or spiketrains
            alltimes = alltrigs['tick'].astype('f')*header.us_per_time * header.dtime_base*pq.s
            
            if channelHeader.kind in [2, 3, 4 , 5 , 8]:
                #events
                ea = EventArray(  )
                ea.annotate(channel_index = channel_num)
                ea.times = alltimes
                if channelHeader.kind >= 5:
                    # Spike2 marker is closer to label sens of neo
                    ea.labels = alltrigs['marker'].astype('S32')
                if channelHeader.kind == 8:
                    ea.annotate(extra_labels = alltrigs['label'])
{'fraction_non_alphanumeric': 0.09645734466394437, 'fraction_numerical': 0.01088915866534231, 'mean_word_length': 1.4465844658446585, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024552', 'n_tokens_mistral': 7863, 'n_tokens_neox': 7389, 'n_words': 1765}