<commit_before>from direct.directnotify.DirectNotifyGlobal import directNotify
from direct.showbase.PythonUtil import invertDict, makeList, safeRepr
from direct.showbase.PythonUtil import getNumberedTypedString
import types
import gc

class Diff:
    def __init__(self, lost, gained):
        self.lost=lost
        self.gained=gained
    def printOut(self):
        print 'lost %s objects, gained %s objects' % (len(self.lost), len(self.gained))
        print '\n\nself.lost\n'
        print self.lost.typeFreqStr()
        print '\n\nself.gained\n'
        print self.gained.typeFreqStr()
        self.gained.printObjsByType()
        print '\n\nGAINED-OBJECT REFERRERS\n'
        print self.gained.referrersStr(1)

class ObjectPool:
    """manipulate a pool of Python objects"""
    notify = directNotify.newCategory('ObjectPool')

    def __init__(self, objects):
        self._objs = list(objects)
        self._type2objs = {}
        self._count2types = {}
        self._len2obj = {}
        type2count = {}
        for obj in self._objs:
            typ = itype(obj)
            type2count.setdefault(typ, 0)
            type2count[typ] += 1
            self._type2objs.setdefault(typ, [])
            self._type2objs[typ].append(obj)
            try:
                self._len2obj[len(obj)] = obj
            except:
                pass
        self._count2types = invertDict(type2count)

    def _getInternalObjs(self):
        return (self._objs, self._type2objs, self._count2types)

    def destroy(self):
        del self._objs
        del self._type2objs
        del self._count2types

    def getTypes(self):
        return self._type2objs.keys()

    def getObjsOfType(self, type):
        return self._type2objs.get(type, [])

    def printObjsOfType(self, type):
        for obj in self._type2objs.get(type, []):
            print repr(obj)

    def diff(self, other):
        """print difference between this pool and 'other' pool"""
        thisId2obj = {}
        otherId2obj = {}
        for obj in self._objs:
            thisId2obj[id(obj)] = obj
        for obj in other._objs:
            otherId2obj[id(obj)] = obj
        thisIds = set(thisId2obj.keys())
        otherIds = set(otherId2obj.keys())
        lostIds = thisIds.difference(otherIds)
        gainedIds = otherIds.difference(thisIds)
        del thisIds
        del otherIds
        lostObjs = []
        for i in lostIds:
            lostObjs.append(thisId2obj[i])
        gainedObjs = []
        for i in gainedIds:
            gainedObjs.append(otherId2obj[i])
        return Diff(self.__class__(lostObjs), self.__class__(gainedObjs))

    def typeFreqStr(self):
        s  =   'Object Pool: Type Frequencies'
        s += '\n============================='
        counts = list(set(self._count2types.keys()))
        counts.sort()
        counts.reverse()
        for count in counts:
            types = makeList(self._count2types[count])
            for typ in types:
                s += '\n%s\t%s' % (count, typ)
        return s

    def printObjsByType(self):
        print 'Object Pool: Objects By Type'
        print '\n============================'
        counts = list(set(self._count2types.keys()))
        counts.sort()
        counts.reverse()
        for count in counts:
            types = makeList(self._count2types[count])
            for typ in types:
                print 'TYPE: %s' % typ
                print getNumberedTypedString(self._type2objs[typ])

    def containerLenStr(self):
        s  =   'Object Pool: Container Lengths'
        s += '\n=============================='
        lengths = list(self._len2obj.keys())
        lengths.sort()
        lengths.reverse()
        for count in counts:
            pass

    def referrersStr(self, numEach=3):
        """referrers of the first few of each type of object"""
        s = ''
        counts = list(set(self._count2types.keys()))
        counts.sort()
        counts.reverse()
        for count in counts:
            types = makeList(self._count2types[count])
            for typ in types:
                s += '\n\nTYPE: %s' % typ
                for i in xrange(min(numEach,len(self._type2objs[typ]))):
                    obj = self._type2objs[typ][i]
                    s += '\nOBJ: %s\n' % safeRepr(obj)
                    referrers = gc.get_referrers(obj)
                    s += '%s REFERRERS:\n' % len(referrers)
                    if len(referrers):
                        s += getNumberedTypedString(referrers, maxLen=80,
                                                    numPrefix='REF')
                    else:
                        s += '<No Referrers>'
        return s

    def __len__(self):
        return len(self._objs)
<commit_msg>fixed count->type dict bug<commit_after>from direct.directnotify.DirectNotifyGlobal import directNotify
from direct.showbase.PythonUtil import invertDictLossless, makeList, safeRepr
from direct.showbase.PythonUtil import getNumberedTypedString
import types
import gc

class Diff:
    def __init__(self, lost, gained):
        self.lost=lost
        self.gained=gained
    def printOut(self):
        print 'lost %s objects, gained %s objects' % (len(self.lost), len(self.gained))
        print '\n\nself.lost\n'
        print self.lost.typeFreqStr()
        print '\n\nself.gained\n'
        print self.gained.typeFreqStr()
        self.gained.printObjsByType()
        print '\n\nGAINED-OBJECT REFERRERS\n'
        self.gained.printReferrers(1)

class ObjectPool:
    """manipulate a pool of Python objects"""
    notify = directNotify.newCategory('ObjectPool')

    def __init__(self, objects):
        self._objs = list(objects)
        self._type2objs = {}
        self._count2types = {}
        self._len2obj = {}
        type2count = {}
        for obj in self._objs:
            typ = itype(obj)
            type2count.setdefault(typ, 0)
            type2count[typ] += 1
            self._type2objs.setdefault(typ, [])
            self._type2objs[typ].append(obj)
            try:
                self._len2obj[len(obj)] = obj
            except:
                pass
        self._count2types = invertDictLossless(type2count)

    def _getInternalObjs(self):
        return (self._objs, self._type2objs, self._count2types)

    def destroy(self):
        del self._objs
        del self._type2objs
        del self._count2types

    def getTypes(self):
        return self._type2objs.keys()

    def getObjsOfType(self, type):
        return self._type2objs.get(type, [])

    def printObjsOfType(self, type):
        for obj in self._type2objs.get(type, []):
            print repr(obj)

    def diff(self, other):
        """print difference between this pool and 'other' pool"""
        thisId2obj = {}
        otherId2obj = {}
        for obj in self._objs:
            thisId2obj[id(obj)] = obj
        for obj in other._objs:
            otherId2obj[id(obj)] = obj
        thisIds = set(thisId2obj.keys())
        otherIds = set(otherId2obj.keys())
        lostIds = thisIds.difference(otherIds)
        gainedIds = otherIds.difference(thisIds)
        del thisIds
        del otherIds
        lostObjs = []
        for i in lostIds:
            lostObjs.append(thisId2obj[i])
        gainedObjs = []
        for i in gainedIds:
            gainedObjs.append(otherId2obj[i])
        return Diff(self.__class__(lostObjs), self.__class__(gainedObjs))

    def typeFreqStr(self):
        s  =   'Object Pool: Type Frequencies'
        s += '\n============================='
        counts = list(set(self._count2types.keys()))
        counts.sort()
        counts.reverse()
        for count in counts:
            types = makeList(self._count2types[count])
            for typ in types:
                s += '\n%s\t%s' % (count, typ)
        return s

    def printObjsByType(self):
        print 'Object Pool: Objects By Type'
        print '\n============================'
        counts = list(set(self._count2types.keys()))
        counts.sort()
        counts.reverse()
        for count in counts:
            types = makeList(self._count2types[count])
            for typ in types:
                print 'TYPE: %s' % repr(typ)
                print getNumberedTypedString(self._type2objs[typ])

    def containerLenStr(self):
        s  =   'Object Pool: Container Lengths'
        s += '\n=============================='
        lengths = list(self._len2obj.keys())
        lengths.sort()
        lengths.reverse()
        for count in counts:
            pass

    def printReferrers(self, numEach=3):
        """referrers of the first few of each type of object"""
        counts = list(set(self._count2types.keys()))
        counts.sort()
        counts.reverse()
        for count in counts:
            types = makeList(self._count2types[count])
            for typ in types:
                print '\n\nTYPE: %s' % repr(typ)
                for i in xrange(min(numEach,len(self._type2objs[typ]))):
                    obj = self._type2objs[typ][i]
                    print '\nOBJ: %s\n' % safeRepr(obj)
                    referrers = gc.get_referrers(obj)
                    print '%s REFERRERS:\n' % len(referrers)
                    if len(referrers):
                        print getNumberedTypedString(referrers, maxLen=80,
                                                    numPrefix='REF')
                    else:
                        print '<No Referrers>'

    def __len__(self):
        return len(self._objs)

{'fraction_non_alphanumeric': 0.12349968414403033, 'fraction_numerical': 0.008843967150979154, 'mean_word_length': 1.8960365853658536, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6519071', 'n_tokens_mistral': 2873, 'n_tokens_neox': 2805, 'n_words': 627}