<commit_before>import os
import re

class MetricPath:
    def __init__(self, parent, name, files):
        self.name = name
        self.parent = parent
        self.files = files
        self.children = None

    def adopt(self, children):
        self.children = children

    def lineage(self):
        line = [self.name,]
        ancestor = self.parent
        while ancestor:
            if ancestor.parent:
                line.insert(0, ancestor.name)
            ancestor = ancestor.parent
        return line

class Index:
    def __init__(self, storage='/opt/graphite/storage/whisper/'):
        self.storage = storage
        self.walk = os.walk(storage)
        self.root = self.create('', None, *self.walk.next())

    def __str__(self):
        return '\n'.join(list(self))

    def __iter__(self):
        """Return sorted list of tree."""
        ls = []
        for metric in self.getList(self.root, ls):
            yield metric
    
    def create(self, name, parent, curdir, subdirs, curfiles):
        newPath = MetricPath(parent,
                name,
                map(lambda x: re.sub('\.wsp$', '', x), curfiles))
        children = []
        for subdir in subdirs:
            children.append(self.create(subdir, newPath, *self.walk.next()))
        newPath.adopt(children)

        return newPath

    def getList(self, root, ls):
        for child in root.children:
            self.getList(child, ls)

        lineage = root.lineage()
        for f in root.files:
            ls.append('.'.join(lineage + [f,]))

        return ls


if __name__ == '__main__':
    i = Index()
    import pdb; pdb.set_trace()<commit_msg>Removed pdb.set_trace()<commit_after>import os
import re

class MetricPath:
    def __init__(self, parent, name, files):
        self.name = name
        self.parent = parent
        self.files = files
        self.children = None

    def adopt(self, children):
        self.children = children

    def lineage(self):
        line = [self.name,]
        ancestor = self.parent
        while ancestor:
            if ancestor.parent:
                line.insert(0, ancestor.name)
            ancestor = ancestor.parent
        return line

class Index:
    def __init__(self, storage='/opt/graphite/storage/whisper/'):
        self.storage = storage
        self.walk = os.walk(storage)
        self.root = self.create('', None, *self.walk.next())

    def __str__(self):
        return '\n'.join(list(self))

    def __iter__(self):
        """Return sorted list of tree."""
        ls = []
        for metric in self.getList(self.root, ls):
            yield metric
    
    def create(self, name, parent, curdir, subdirs, curfiles):
        newPath = MetricPath(parent,
                name,
                map(lambda x: re.sub('\.wsp$', '', x), curfiles))
        children = []
        for subdir in subdirs:
            children.append(self.create(subdir, newPath, *self.walk.next()))
        newPath.adopt(children)

        return newPath

    def getList(self, root, ls):
        for child in root.children:
            self.getList(child, ls)

        lineage = root.lineage()
        for f in root.files:
            ls.append('.'.join(lineage + [f,]))

        return ls


if __name__ == '__main__':
    i = Index()
{'fraction_non_alphanumeric': 0.1211104331909701, 'fraction_numerical': 0.0006101281269066504, 'mean_word_length': 1.904340124003543, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4014511', 'n_tokens_mistral': 1010, 'n_tokens_neox': 965, 'n_words': 249}