<commit_before>from ROOT import TTree, TFile, TH1D, TH2D, TH3D, MethodProxy, gInterpreter
from IPython.display import HTML, Markdown, display
from time import time
from array import array
import ROOT
import sys
from PyTreeReader import PyTreeReader
## TODO - New file for mapping and another tree from that
## TODO - New File for flatmapping

class tree(object):
    def __init__(self, tree):
        #self.filename = file
        #self.treename = tree
        #self.file = TFile(self.filename)
        #self.tree = self.file.__getattr__(self.treename)
        self.tree = tree
        self.names = [b.GetName() for b in self.tree.GetListOfBranches()]
        self.createPyTreeReader(tree)
        self.filters = []
        self.c_filters = []

        self.cache_c_filters = []
        self.cache_filters = []

        self.maps = []
        self.cache_maps = []
        self.flatMaps = []
        self.cache_flatMaps = []
        self.non_cached_transformations = []
        self.cached_transformations = []

        self.newCacheFilters = False

        self.newCache = False
        self.hvalue = 0
        self.useCache = False
        self.cacheEntryList = []

        self.caching = False
        self.wrappers = {}
        #Uses this for mapping and flatmapping
        self.cachedTree = None
        #self.PyTreeReader = None

        # TODO TODO FIX THIS REGARDING HISTO
    def head(self, rows = 5):
        reader =  self.PyTreeReader
        names = self.names
        text  = '|' + '|'.join(names) + '|\n'
        text += '|' + '|'.join('---' for n in names) + '|\n'
        i = 0
        if self.useCache:
            print "USES CACHED DATA"
            position = 0
            for entry in reader:
                if self.testEntryList.Contains(position):
                    if self.__apply_filters:
                        if self.__apply_filters(entry) : continue
                        text += '|' + '|'.join(str(entry.__getattr__(n)()) for n in names) + '|\n'
                        i += 1
                        if i >= rows : break
                    else:
                        text += '|' + '|'.join(str(entry.__getattr__(n)()) for n in names) + '|\n'
                        i += 1
                        if i >= rows : break
                position += 1
        else:
            for entry in reader:
                    if self.__apply_filters(entry) : continue
                    text += '|' + '|'.join(str(entry.__getattr__(n)()) for n in names) + '|\n'
                    i += 1
                    if i >= rows : break
        self.__reset_filters()
        display(Markdown(text))

# -- Apply Transformations
    def __apply_filter(self, func, entry):
        #We need this if we want to use the correct order and apply only 1 filter at a time.
        #What should this return?
        filtered = False
        #for func in self.cache_filters:
        if func(entry) :
            filtered = True
                #break
        return filtered

    def __apply_filters(self, entry):
        filtered = False
        for f in self.filters:
            if not f(entry) :
                filtered = True
                break
        return filtered

    #TODO CHECK IF THIS IS DOABLE LIKE THIS!
    def __apply_map(self):
        mapped = False
        maplist = self.maps
        for m in maplist:
            self.tree = map(m, self.tree)
            print self.tree
            # TODO here maybe use friendTree and return that? always create a new friendTree when applying map?
            # If its found then always  rewrite / use it instead of normal self.tree.
            # The functionality of filters has to be changed also

            # We could also first run it so that is changes the self.tree
            # Then step by step implement friendTree because it has to be implemented in many places.

            #if not m(tree):
            mapped = True
            break
        return mapped

    def __apply_map_cached(self, func):
        # Creates a new tree, and applies a new column to that with mapped values.
        # Currently it creates new file (Tree) and adds a new column to it.
        # It calculates the new values but it doesnt fill the branch with it.
        # TODO FIX APPLYING VALUES TO THE BRANCH

        #Another way would be creating just a tree with only this one mapped value
        mapped = False
        myvar = array( 'i', [ 0 ] )
        #if self.cachedTree = None:
        events = self.tree.GetEntries()
        newfile = TFile("newMappedFile.root","RECREATE")
        newtree = self.tree.CloneTree(0)
        leafValues = map(func, self.tree)
        #else:
        #    events = self.cachedTree.GetEntries()
        #    newfile = TFile("newMappedFile.root","RECREATE")
        #    newtree = self.cachedTree.CloneTree(0)
        #    leafValues = map(func, self.cachedTree)
        #myvar = array(leafValues)
        listofBranch = newtree.GetListOfBranches()

        #newBranch = newtree.Branch( "new_vars" , leafValues, leaves )
        newBranch = newtree.Branch("mappedVal", myvar, 'mappedVal/I')
        for b in listofBranch:
            print b
        #for i in range(events):
            #self.tree.GetEntry(i)
        for val in leafValues:
            #newtree.Branch('myvar')
            newtree.mappedVal = val
            #print newtree.myvar
            #newtree.getBranch('myvar')
            #fill the new column here with mapped values
            newtree.Fill()
        newtree.Write()
        self.cachedTree = newtree

        print "Saved tree"
        #self.tree.Branch( 'testfirstVar', mystruct, 'testsecondVar' )
        #print maptree
        #self.file = TFile(self.filename)
        #self.tree = self.file.__getattr__(self.treename)
        #mappedTreeFile = TFile(self.filename, 'RECREATE')
        #mappedTree = self.file.__getattr__(self.treename)
        #for val in maptree:
        #    mappedTree.fMyTest = val
        #    mappedTree.Fill()
        #mappedTree.Write()
        #mappedTreeFile.Close()
        mapped = True
        #print mappedTree
        #break
        return mapped

# -- Reset lists for the next run
    def __reset_maps(self):
        self.maps = []
        self.mapvalue = 0
        self.flatMaps = []

    def __reset_filters(self):
        self.filters = []
        self.non_cached_transformations = []

# -- You can reset the cache with this command
    def resetcache(self):
        #Reset cache for testing purposes
        self.useCache = False
        self.cache_filters = []

        self.filters = []
        self.cacheEntryList = []

        self.hvalue = 0

        self.mapvalue = 0
        self.maps = []
        self.cache_maps = []

        self.flatMaps = []
        self.cache_flatMaps = []

        return self

# -- Transformations functions add the functions to list
    def flatMap(self, func):
        self.flatMaps.append(func)
        self.non_cached_transformations.append(func)
        return self

    def map(self, func):
        self.maps.append(func)
        self.non_cached_transformations.append(func)
        return self

    def filter(self, func):
        self.filters.append(func)
        self.non_cached_transformations.append(func)
        return self

# -- End of Transformations
    def cache(self):
        reader = self.PyTreeReader
        #Have to compare __code__ functions of the functions for identification
        non_cached_transformationsList = self.non_cached_transformations<commit_msg>clean-up<commit_after>from ROOT import TTree, TFile, TH1D, TH2D, TH3D, MethodProxy, gInterpreter
from IPython.display import HTML, Markdown, display
from time import time
from array import array
import ROOT
import sys
from PyTreeReader import PyTreeReader
## TODO - New file for mapping and another tree from that
## TODO - New File for flatmapping

class tree(object):
    def __init__(self, tree):
        self.tree = tree
        self.names = [b.GetName() for b in self.tree.GetListOfBranches()]
        self.createPyTreeReader(tree)
        self.filters = []
        self.cache_filters = []

        self.maps = []
        self.cache_maps = []
        self.flatMaps = []
        self.cache_flatMaps = []
        self.non_cached_transformations = []
        self.cached_transformations = []

        self.newCacheFilters = False

        self.newCache = False
        self.useCache = False
        self.cacheEntryList = []
        #Uses this for mapping and flatmapping
        self.cachedTree = None
        #self.PyTreeReader = None

        # TODO TODO FIX THIS REGARDING HISTO
    def head(self, rows = 5):
        reader =  self.PyTreeReader
        names = self.names
        text  = '|' + '|'.join(names) + '|\n'
        text += '|' + '|'.join('---' for n in names) + '|\n'
        i = 0
        if self.useCache:
            print "USES CACHED DATA"
            position = 0
            for entry in reader:
                if self.testEntryList.Contains(position):
                    if self.__apply_filters:
                        if self.__apply_filters(entry) : continue
                        text += '|' + '|'.join(str(entry.__getattr__(n)()) for n in names) + '|\n'
                        i += 1
                        if i >= rows : break
                    else:
                        text += '|' + '|'.join(str(entry.__getattr__(n)()) for n in names) + '|\n'
                        i += 1
                        if i >= rows : break
                position += 1
        else:
            for entry in reader:
                    if self.__apply_filters(entry) : continue
                    text += '|' + '|'.join(str(entry.__getattr__(n)()) for n in names) + '|\n'
                    i += 1
                    if i >= rows : break
        self.__reset_filters()
        display(Markdown(text))

# -- Apply Transformations
    def __apply_filter(self, func, entry):
        filtered = False
        if func(entry) :
            filtered = True
        return filtered

    def __apply_filters(self, entry):
        filtered = False
        for f in self.filters:
            if not f(entry) :
                filtered = True
                break
        return filtered

    #TODO CHECK IF THIS IS DOABLE LIKE THIS!
    def __apply_map(self):
        mapped = False
        maplist = self.maps
        for m in maplist:
            self.tree = map(m, self.tree)
            print self.tree
            # TODO here maybe use friendTree and return that? always create a new friendTree when applying map?
            # If its found then always  rewrite / use it instead of normal self.tree.
            # The functionality of filters has to be changed also

            # We could also first run it so that is changes the self.tree
            # Then step by step implement friendTree because it has to be implemented in many places.

            #if not m(tree):
            mapped = True
            break
        return mapped

    def __apply_map_cached(self, func):
        # Creates a new tree, and applies a new column to that with mapped values.
        # Currently it creates new file (Tree) and adds a new column to it.
        # It calculates the new values but it doesnt fill the branch with it.
        # TODO FIX APPLYING VALUES TO THE BRANCH

        #Another way would be creating just a tree with only this one mapped value
        mapped = False
        myvar = array( 'i', [ 0 ] )
        #if self.cachedTree = None:
        events = self.tree.GetEntries()
        newfile = TFile("newMappedFile.root","RECREATE")
        newtree = self.tree.CloneTree(0)
        leafValues = map(func, self.tree)
        #else:
        #    events = self.cachedTree.GetEntries()
        #    newfile = TFile("newMappedFile.root","RECREATE")
        #    newtree = self.cachedTree.CloneTree(0)
        #    leafValues = map(func, self.cachedTree)
        #myvar = array(leafValues)
        listofBranch = newtree.GetListOfBranches()

        #newBranch = newtree.Branch( "new_vars" , leafValues, leaves )
        newBranch = newtree.Branch("mappedVal", myvar, 'mappedVal/I')
        for b in listofBranch:
            print b
        #for i in range(events):
            #self.tree.GetEntry(i)
        for val in leafValues:
            #newtree.Branch('myvar')
            newtree.mappedVal = val
            #print newtree.myvar
            #newtree.getBranch('myvar')
            #fill the new column here with mapped values
            newtree.Fill()
        newtree.Write()
        self.cachedTree = newtree

        print "Saved tree"
        #self.tree.Branch( 'testfirstVar', mystruct, 'testsecondVar' )
        #print maptree
        #self.file = TFile(self.filename)
        #self.tree = self.file.__getattr__(self.treename)
        #mappedTreeFile = TFile(self.filename, 'RECREATE')
        #mappedTree = self.file.__getattr__(self.treename)
        #for val in maptree:
        #    mappedTree.fMyTest = val
        #    mappedTree.Fill()
        #mappedTree.Write()
        #mappedTreeFile.Close()
        mapped = True
        #print mappedTree
        #break
        return mapped

# -- Reset lists for the next run
    def __reset_maps(self):
        self.maps = []
        self.mapvalue = 0
        self.flatMaps = []

    def __reset_filters(self):
        self.filters = []
        self.non_cached_transformations = []

# -- You can reset the cache with this command
    def resetcache(self):
        #Reset cache for testing purposes
        self.useCache = False
        self.cache_filters = []

        self.filters = []
        self.cacheEntryList = []

        self.mapvalue = 0
        self.maps = []
        self.cache_maps = []

        self.flatMaps = []
        self.cache_flatMaps = []

        return self

# -- Transformations functions add the functions to list
    def flatMap(self, func):
        self.flatMaps.append(func)
        self.non_cached_transformations.append(func)
        return self

    def map(self, func):
        self.maps.append(func)
        self.non_cached_transformations.append(func)
        return self

    def filter(self, func):
        self.filters.append(func)
        self.non_cached_transformations.append(func)
        return self

# -- End of Transformations
    def cache(self):
        reader = self.PyTreeReader
        #Have to compare __code__ functions of the functions for identification
        non_cached_transformationsList = self.non_cached_transformations
{'fraction_non_alphanumeric': 0.0917513655534813, 'fraction_numerical': 0.0022816842978635138, 'mean_word_length': 1.806908596933825, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1630826', 'n_tokens_mistral': 4083, 'n_tokens_neox': 4008, 'n_words': 1188}