<commit_before>
import re

from . import nodes


class _ParsingSentinal(object):
    def __init__(self, *args, **kwargs):
        self.__dict__.update(**kwargs)

INC_DEPTH = _ParsingSentinal()

class Parser(object):
    
    def __init__(self):
        self.root = nodes.Document()
        self.stack = [(-1, self.root)]
        self._peeked = None
    
    @property
    def depth(self):
        return self.stack[-1][0]
        
    @property
    def node(self):
        return self.stack[-1][1]
    
    def parse_string(self, source):
        self.parse(source.splitlines())
    
    def next_line(self):
        if self._peeked is not None:
            line = self._peeked
            self._peeked = None
            return line
        return next(self.source)
    
    def peek_line(self):
        if self._peeked is None:
            self._peeked = next(self.source)
        return self._peeked
    
    def parse(self, source):
        self.source = iter(source)
        while True:
            try:
                raw_line = self.next_line()
            except StopIteration:
                return
            try:
                while raw_line.endswith('|'):
                    raw_line = raw_line[:-1]
                    if self.peek_line().endswith('|'):
                        raw_line += self.next_line()
            except StopIteration:
                pass
            line = raw_line.lstrip()
            if not line:
                continue
            depth = len(raw_line) - len(line)
            # Cleanup the stack. We should only need to do this here as the
            # depth only goes up until it is calculated from the next line.
            self.prep_stack_for_depth(depth)
            last_line = None
            while line and line != last_line:
                last_line = line
                for token in self._parse_line(line):
                    if isinstance(token, nodes.Base):
                        self.add_node(token, depth)
                    elif isinstance(token, basestring):
                        line = token
                        depth += 1
                    else:
                        raise TypeError('unknown token type %r' % token)
                
    
    def _parse_line(self, line):
        
        if isinstance(self.node, nodes.GreedyBase):
            yield self.node.__class__.with_parent(self.node, line)
            return
        
        # Escape a line so it doesn't get touched.
        if line.startswith('\\'):
            yield nodes.Content(line[1:].lstrip())
            return
        
        # HTML comments.
        m = re.match(r'/(\[if[^\]]+])?(.*)$', line)
        if m:<commit_msg>Inter- and intra-line depths are tracked seperate.<commit_after>
import re

from . import nodes


class _ParsingSentinal(object):
    def __init__(self, *args, **kwargs):
        self.__dict__.update(**kwargs)

INC_DEPTH = _ParsingSentinal()

class Parser(object):
    
    def __init__(self):
        self.root = nodes.Document()
        self.stack = [((-1, 0), self.root)]
        self._peeked = None
    
    @property
    def depth(self):
        return self.stack[-1][0]
        
    @property
    def node(self):
        return self.stack[-1][1]
    
    def parse_string(self, source):
        self.parse(source.splitlines())
    
    def next_line(self):
        if self._peeked is not None:
            line = self._peeked
            self._peeked = None
            return line
        return next(self.source)
    
    def peek_line(self):
        if self._peeked is None:
            self._peeked = next(self.source)
        return self._peeked
    
    def parse(self, source):
        self.source = iter(source)
        while True:
            try:
                raw_line = self.next_line()
            except StopIteration:
                return
            try:
                while raw_line.endswith('|'):
                    raw_line = raw_line[:-1]
                    if self.peek_line().endswith('|'):
                        raw_line += self.next_line()
            except StopIteration:
                pass
            line = raw_line.lstrip()
            if not line:
                continue
            # We track the inter-line depth seperate from the intra-line depth
            # so that indentation due to whitespace always results in more
            # depth in the graph than many nested nodes from a single line.
            inter_depth = len(raw_line) - len(line)
            intra_depth = 0
            # Cleanup the stack. We should only need to do this here as the
            # depth only goes up until it is calculated from the next line.
            self.prep_stack_for_depth((inter_depth, 0))
            last_line = None
            while line and line != last_line:
                last_line = line
                for token in self._parse_line(line):
                    if isinstance(token, nodes.Base):
                        self.add_node(token, (inter_depth, intra_depth))
                    elif isinstance(token, basestring):
                        line = token
                        intra_depth += 1
                    else:
                        raise TypeError('unknown token type %r' % token)
                
    
    def _parse_line(self, line):
        
        if isinstance(self.node, nodes.GreedyBase):
            yield self.node.__class__.with_parent(self.node, line)
            return
        
        # Escape a line so it doesn't get touched.
        if line.startswith('\\'):
            yield nodes.Content(line[1:].lstrip())
            return
        
        # HTML comments.
        m = re.match(r'/(\[if[^\]]+])?(.*)$', line)
        if m:
{'fraction_non_alphanumeric': 0.0937281009110021, 'fraction_numerical': 0.0033286615276804485, 'mean_word_length': 1.4190677966101695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2429532', 'n_tokens_mistral': 1605, 'n_tokens_neox': 1533, 'n_words': 416}