<commit_before>'''
Created on Apr 15, 2013

@author: timmahrt
'''

import re
import copy
import functools

import codecs

from os.path import join

# Can only handle interval tiers at the moment
INTERVAL_TIER = "IntervalTier"


def _getMinInTupleList(timestampList):
    return min([float(subList[0]) for subList in timestampList])


def _getMaxInTupleList(timestampList):
    return max([float(subList[1]) for subList in timestampList])


def _morphFunc(fromTier, toTier):
    for fromEntry, toEntry in zip(fromTier.entryList, toTier.entryList):
        
        fromStart, fromEnd, fromLabel = fromEntry
        toStart, toEnd, toLabel = toEntry
        
        # Silent pauses are not manipulated to the target destination
        if fromLabel == 'sp' or fromLabel == '':
            tmpStart = fromStart
            tmpEnd = fromEnd
        else:
            tmpStart = toStart
            tmpEnd = toEnd

        yield tmpStart, tmpEnd, fromLabel


def _manipulateFunc(fromTier, modFunc, filterFunc):
    for fromEntry in fromTier.entryList:
        
        fromStart, fromEnd, fromLabel = fromEntry
        
        # Silent pauses are not manipulated to the target destination
        if fromLabel == 'sp' or fromLabel == '' or not filterFunc(fromLabel):
            tmpStart = fromStart
            tmpEnd = fromEnd
        else:
            tmpStart, tmpEnd = modFunc(fromStart, fromEnd)
            
        yield tmpStart, tmpEnd, fromLabel


def _manipulate(tier, iterateFunc):
    '''
    A generic function for manipulating tiers
    
    The provided /iterateFunc/ specifies the new values for old textgrid regions
    
    The job of this function is to determine the new location of each textgrid
    intervals (taking into account the other textgrid intervals)
    '''
    adjustedEntryList = []
    adjustAmount = 0.0 # Chains adjustments from prior manipulations onto later ones
    for tmpStart, tmpEnd, fromLabel in iterateFunc():

        tmpAdjustAmount = (tmpEnd - tmpStart)
        
        adjustedStart = adjustAmount
        adjustedEnd = adjustAmount + tmpAdjustAmount
        
        adjustAmount += tmpAdjustAmount
        
        adjustedEntryList.append( [adjustedStart, adjustedEnd, fromLabel] )
        
    adjustedTier = TextgridTier(tier.name, adjustedEntryList, tier.tierType)           
    
    return adjustedTier


class TimelessTextgridTierException(Exception):
    
    def __str__(self):
        return "All textgrid tiers much have a min and max duration"


class BadIntervalError(Exception):
    
    def __init__(self, start, stop, label):
        self.start = start
        self.stop = stop
        self.label = label
        
    def __str__(self):
        return "Problem with interval--could not create textgrid (%s,%s,%s)" % (self.start, self.stop, self.label)


class TextgridTier():
    
    
    def __init__(self, name, entryList, tierType, minT=None, maxT=None):
        
        # Prevent poorly-formed textgrids from being created
        for entry in entryList:
            if float(entry[0]) > float(entry[1]):
                print "Anomaly: startTime=%f, stopTime=%f, label=%s" % (entry[0], entry[1], entry[2])
            assert(float(entry[0]) < float(entry[1]))

        
        # Remove whitespace
        tmpEntryList = []
        for start, stop, label in entryList:
            tmpEntryList.append( (start, stop, label.strip()))
        entryList = tmpEntryList
        
        self.name = name
        self.entryList = entryList
        self.tierType = tierType
        
        if minT != None and maxT != None:
            if entryList == None or entryList == []:
                entryList.append( [minT, maxT, ""] )
        elif entryList != None and entryList != []:
            minT = _getMinInTupleList(entryList)
            maxT = _getMaxInTupleList(entryList)
        else:
            # Need to have some timing information to create a textgrid tier
            raise TimelessTextgridTierException()
            
        self.minTimestamp = minT
        self.maxTimestamp = maxT
        
        # If an entry list contains "holes"-- fill them in
        self.entryList = self._fillInBlanks("", self.minTimestamp, self.maxTimestamp)
    
    
    def insertInterval(self, startTime, endTime, label):
        
        # True if any existing times overlap with the new insertion time
        matchList = [ (float(startTime) <= float(oldEndTime) and
                    float(endTime) >= float(oldStartTime)) for oldStartTime, oldEndTime, oldLabel in self.entryList]
        print matchList
        print matchList.count(True)
        print self.entryList[matchList.index(True)]
        assert(matchList.count(True) == 0 or (matchList.count(True) == 1 and self.entryList[matchList.index(True)][2] == ""))
        
        # For inserting into an 'empty' textgrid
        if matchList.count(True) == 0:
            self.entryList.append([startTime, endTime, label])
            self.entryList.sort()
            
        # For inserting into a textgrid with content
        elif matchList.count(True) == 1:
            i = matchList.index(True)
            oldStart, oldStop, oldLabel = self.entryList[i]
        
            subEntryList = []
            if oldStart != startTime:
                subEntryList.append([oldStart, startTime, ""])
            subEntryList.append([startTime, endTime, label])
            if oldStop != endTime:
                subEntryList.append([endTime, oldStop, ""]) 
    
            newEntryList = self.entryList[:i] + subEntryList + self.entryList[i+1:]
            self.entryList = newEntryList
            
    
    def getDuration(self):
        return self.maxTimestamp - self.minTimestamp
    
    <commit_msg>REFACTOR: Pulled out the overlap check code so everyone can use it<commit_after>'''
Created on Apr 15, 2013

@author: timmahrt
'''

import re
import copy
import functools

import codecs

from os.path import join

# Can only handle interval tiers at the moment
INTERVAL_TIER = "IntervalTier"


def _getMinInTupleList(timestampList):
    return min([float(subList[0]) for subList in timestampList])


def _getMaxInTupleList(timestampList):
    return max([float(subList[1]) for subList in timestampList])


def _morphFunc(fromTier, toTier):
    for fromEntry, toEntry in zip(fromTier.entryList, toTier.entryList):
        
        fromStart, fromEnd, fromLabel = fromEntry
        toStart, toEnd, toLabel = toEntry
        
        # Silent pauses are not manipulated to the target destination
        if fromLabel == 'sp' or fromLabel == '':
            tmpStart = fromStart
            tmpEnd = fromEnd
        else:
            tmpStart = toStart
            tmpEnd = toEnd

        yield tmpStart, tmpEnd, fromLabel


def _manipulateFunc(fromTier, modFunc, filterFunc):
    for fromEntry in fromTier.entryList:
        
        fromStart, fromEnd, fromLabel = fromEntry
        
        # Silent pauses are not manipulated to the target destination
        if fromLabel == 'sp' or fromLabel == '' or not filterFunc(fromLabel):
            tmpStart = fromStart
            tmpEnd = fromEnd
        else:
            tmpStart, tmpEnd = modFunc(fromStart, fromEnd)
            
        yield tmpStart, tmpEnd, fromLabel


def _manipulate(tier, iterateFunc):
    '''
    A generic function for manipulating tiers
    
    The provided /iterateFunc/ specifies the new values for old textgrid regions
    
    The job of this function is to determine the new location of each textgrid
    intervals (taking into account the other textgrid intervals)
    '''
    adjustedEntryList = []
    adjustAmount = 0.0 # Chains adjustments from prior manipulations onto later ones
    for tmpStart, tmpEnd, fromLabel in iterateFunc():

        tmpAdjustAmount = (tmpEnd - tmpStart)
        
        adjustedStart = adjustAmount
        adjustedEnd = adjustAmount + tmpAdjustAmount
        
        adjustAmount += tmpAdjustAmount
        
        adjustedEntryList.append( [adjustedStart, adjustedEnd, fromLabel] )
        
    adjustedTier = TextgridTier(tier.name, adjustedEntryList, tier.tierType)           
    
    return adjustedTier


def _intervalOverlapCheck(interval, cmprInterval):
    '''Checks whether two intervals overlap'''
    
    startTime, endTime, label = interval
    cmprStartTime, cmprEndTime, cmprLabel = cmprInterval
    
    return (float(startTime) <= float(cmprEndTime) and 
            float(endTime) >= float(cmprStartTime))


class TimelessTextgridTierException(Exception):
    
    def __str__(self):
        return "All textgrid tiers much have a min and max duration"


class BadIntervalError(Exception):
    
    def __init__(self, start, stop, label):
        self.start = start
        self.stop = stop
        self.label = label
        
    def __str__(self):
        return "Problem with interval--could not create textgrid (%s,%s,%s)" % (self.start, self.stop, self.label)


class TextgridTier():
    
    
    def __init__(self, name, entryList, tierType, minT=None, maxT=None):
        
        # Prevent poorly-formed textgrids from being created
        for entry in entryList:
            if float(entry[0]) > float(entry[1]):
                print "Anomaly: startTime=%f, stopTime=%f, label=%s" % (entry[0], entry[1], entry[2])
            assert(float(entry[0]) < float(entry[1]))

        
        # Remove whitespace
        tmpEntryList = []
        for start, stop, label in entryList:
            tmpEntryList.append( (start, stop, label.strip()))
        entryList = tmpEntryList
        
        self.name = name
        self.entryList = entryList
        self.tierType = tierType
        
        if minT != None and maxT != None:
            if entryList == None or entryList == []:
                entryList.append( [minT, maxT, ""] )
        elif entryList != None and entryList != []:
            minT = _getMinInTupleList(entryList)
            maxT = _getMaxInTupleList(entryList)
        else:
            # Need to have some timing information to create a textgrid tier
            raise TimelessTextgridTierException()
            
        self.minTimestamp = minT
        self.maxTimestamp = maxT
        
        # If an entry list contains "holes"-- fill them in
        self.entryList = self._fillInBlanks("", self.minTimestamp, self.maxTimestamp)
    
    
    def insertInterval(self, startTime, endTime, label):
        interval = (startTime, endTime, label)
        
        # True if any existing times overlap with the new insertion time
        matchList = [ _intervalOverlapCheck(interval, oldInterval) for oldInterval in self.entryList]
        print matchList
        print matchList.count(True)
        print self.entryList[matchList.index(True)]
        assert(matchList.count(True) == 0 or (matchList.count(True) == 1 and self.entryList[matchList.index(True)][2] == ""))
        
        # For inserting into an 'empty' textgrid
        if matchList.count(True) == 0:
            self.entryList.append([startTime, endTime, label])
            self.entryList.sort()
            
        # For inserting into a textgrid with content
        elif matchList.count(True) == 1:
            i = matchList.index(True)
            oldStart, oldStop, oldLabel = self.entryList[i]
        
            subEntryList = []
            if oldStart != startTime:
                subEntryList.append([oldStart, startTime, ""])
            subEntryList.append([startTime, endTime, label])
            if oldStop != endTime:
                subEntryList.append([endTime, oldStop, ""]) 
    
            newEntryList = self.entryList[:i] + subEntryList + self.entryList[i+1:]
            self.entryList = newEntryList
            
    
    def getDuration(self):
        return self.maxTimestamp - self.minTimestamp
    
    
{'fraction_non_alphanumeric': 0.08542160514730782, 'fraction_numerical': 0.0038943447341686422, 'mean_word_length': 2.2896129211918685, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2304314', 'n_tokens_mistral': 3329, 'n_tokens_neox': 3219, 'n_words': 940}