<commit_before>class Reader(object):
    def __init__(self, text):
        self.text = text

        self.padded = None
        self.exploded = None
        self.cleaned = None
        self.scanned = None
        self.collapsed = None

        self.pad()
        self.explode()
        self.clean()
        self.scan()
        self.collapse()
        self.finalize()

    def get(self):
        return list(self.collapsed)

    def pad(self):
        text = self.text
        subs = [
            ("(", " ( "),
            (")", " ) "),
            ("\n", "  "),
            ("\t", "  ")
        ]

        for org, new in subs:
            text = text.replace(org, new)

        self.padded = text

    def explode(self):
        self.exploded = self.padded.split(" ")

    def clean(self):
        self.cleaned = filter(None, self.exploded)

    def scan(self):
        self.scanned = []
        start = []

        for i, symbol in enumerate(self.cleaned):
            if symbol is '(':
                start.append(i)
            elif symbol is ')':
                self.scanned.append((start.pop(),i+1))

    def collapse(self):
        self.collapsed = list(self.cleaned)

        for start, end in self.scanned:
            self.collapsed[start:end] = [self.collapsed[start:end]] + \
                                        [None] * (end - start - 1)

    def finalize(self):
        stack = [self.collapsed]

        while stack:
            target = stack.pop()
            items = []

            for i, item in enumerate(target):
                if item is None:
                    items.append(i)
                if isinstance(item, list):
                    stack.append(item)

            items.reverse()

            for i in items:
                target.pop(i)

            try:
                target.remove('(')
                target.remove(')')
            except ValueError:
                pass

        self.collapsed = self.collapsed.pop()
<commit_msg>Cause the reader to work during get(), not before.<commit_after>class Reader(object):
    def __init__(self, text):
        self.text = text

        self.padded = None
        self.exploded = None
        self.cleaned = None
        self.scanned = None
        self.collapsed = None

    def get(self):
        self.pad()
        self.explode()
        self.clean()
        self.scan()
        self.collapse()

        return self.finalize()

    def pad(self):
        text = self.text
        subs = [
            ("(", " ( "),
            (")", " ) "),
            ("\n", "  "),
            ("\t", "  ")
        ]

        for org, new in subs:
            text = text.replace(org, new)

        self.padded = text

    def explode(self):
        self.exploded = self.padded.split(" ")

    def clean(self):
        self.cleaned = filter(None, self.exploded)

    def scan(self):
        self.scanned = []
        start = []

        for i, symbol in enumerate(self.cleaned):
            if symbol is '(':
                start.append(i)
            elif symbol is ')':
                self.scanned.append((start.pop(),i+1))

    def collapse(self):
        self.collapsed = list(self.cleaned)

        for start, end in self.scanned:
            self.collapsed[start:end] = [self.collapsed[start:end]] + \
                                        [None] * (end - start - 1)

    def finalize(self):
        stack = [self.collapsed]

        while stack:
            target = stack.pop()
            items = []

            for i, item in enumerate(target):
                if item is None:
                    items.append(i)
                if isinstance(item, list):
                    stack.append(item)

            items.reverse()

            for i in items:
                target.pop(i)

            try:
                target.remove('(')
                target.remove(')')
            except ValueError:
                pass

        return self.collapsed.pop()

{'fraction_non_alphanumeric': 0.12360970677451971, 'fraction_numerical': 0.0010111223458038423, 'mean_word_length': 1.2508532423208192, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2047147', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1080, 'n_words': 251}