<commit_before>from __future__ import absolute_import
from .lexer import Lexer
from . import nodes
import six

class Parser(object):
    def __init__(self,str,filename=None,**options):
        self.input = str
        self.lexer = Lexer(str,**options)
        self.filename = filename
        self.bloks = {}
        self.options = options
        self.contexts = [self]
        self.extending = False
        self._spaces = None

    def context(self,parser):
        if parser: self.context.append(parser)
        else: self.contexts.pop()

    def advance(self):
        return self.lexer.advance()

    def skip(self,n):
        while n>1: # > 0?
            self.advance()
            n -= 1

    def peek(self):
        p = self.lookahead(1)
        return p

    def line(self):
        return self.lexer.lineno

    def lookahead(self,n):
        return self.lexer.lookahead(n)

    def parse(self):
        block = nodes.Block()
        parser = None
        block.line = self.line()

        while 'eos' != self.peek().type:
            if 'newline' == self.peek().type: self.advance()
            else: block.append(self.parseExpr())

        parser = self.extending
        if parser:
            self.context(parser)
            ast = parser.parse()
            self.context()
            return ast

        return block

    def expect(self,type):
        t = self.peek().type
        if t == type: return self.advance()
        else:
            raise Exception('expected "%s" but got "%s" in file %s on line %d' %
                            (type, t, self.filename, self.line()))

    def accept(self,type):
        if self.peek().type == type: return self.advance()

    def parseExpr(self):
        t = self.peek().type
        if 'yield' == t:
            self.advance()
            block = nodes.Block()
            block._yield = True
            return block
        elif t in ('id','class'):
            tok = self.advance()
            new_div = self.lexer.tok('tag','div')
            new_div.inline_level = tok.inline_level
            self.lexer.stash.append(new_div)
            self.lexer.stash.append(tok)
            return self.parseExpr()

        funcName = 'parse%s'%t.capitalize()
        if hasattr(self,funcName):
            return getattr(self,funcName)()
        else:
            raise Exception('unexpected token "%s" in file %s on line %d' %
                            (t, self.filename, self.line()))

    def parseString(self):
        tok = self.expect('string')
        node = nodes.String(tok.val, inline=tok.inline_level > 0)
        node.line = self.line()
        return node

    def parseText(self):
        tok = self.expect('text')
        node = nodes.Text(tok.val)
        node.line = self.line()
        return node

    def parseBlockExpansion(self):
        if ':'== self.peek().type:
            self.advance()
            return nodes.Block(self.parseExpr())
        else:
            return self.block()

    def parseAssignment(self):
        tok = self.expect('assignment')
        return nodes.Assignment(tok.name,tok.val)

    def parseCode(self):
        tok = self.expect('code')
        if tok.buffer:
            node = nodes.Code(tok.val,True,tok.escape) #tok.escape
            block,i = None,1
            node.line = self.line()
            while self.lookahead(i) and 'newline'==self.lookahead(i).type:
                i+= 1
            block = 'indent' == self.lookahead(i).type
            if block:
                self.skip(i-1)
                node.block = self.block()
        else:
            if 'indent'==self.peek().type:
                node = nodes.BlockCode(
                    tok.val, self.parseTextBlock(), tok.buffer, tok.escape)
            else:
                node = nodes.Code(tok.val,tok.buffer, escape)

            node.line = self.line()
        return node

    def parseComment(self):
        tok = self.expect('comment')

        if 'indent'==self.peek().type:
            node = nodes.BlockComment(tok.val, self.block(), tok.buffer)
        else:
            node = nodes.Comment(tok.val,tok.buffer)

        node.line = self.line()
        return node

    def parseDoctype(self):
        tok = self.expect('doctype')
        node = nodes.Doctype(tok.val)
        node.line = self.line()<commit_msg>Keep backward compatibility<commit_after>from __future__ import absolute_import
from .lexer import Lexer
from . import nodes
import six

class Parser(object):
    def __init__(self,str,filename=None,**options):
        self.input = str
        self.lexer = Lexer(str,**options)
        self.filename = filename
        self.bloks = {}
        self.options = options
        self.contexts = [self]
        self.extending = False
        self._spaces = None

    def context(self,parser):
        if parser: self.context.append(parser)
        else: self.contexts.pop()

    def advance(self):
        return self.lexer.advance()

    def skip(self,n):
        while n>1: # > 0?
            self.advance()
            n -= 1

    def peek(self):
        p = self.lookahead(1)
        return p

    def line(self):
        return self.lexer.lineno

    def lookahead(self,n):
        return self.lexer.lookahead(n)

    def parse(self):
        block = nodes.Block()
        parser = None
        block.line = self.line()

        while 'eos' != self.peek().type:
            if 'newline' == self.peek().type: self.advance()
            else: block.append(self.parseExpr())

        parser = self.extending
        if parser:
            self.context(parser)
            ast = parser.parse()
            self.context()
            return ast

        return block

    def expect(self,type):
        t = self.peek().type
        if t == type: return self.advance()
        else:
            raise Exception('expected "%s" but got "%s" in file %s on line %d' %
                            (type, t, self.filename, self.line()))

    def accept(self,type):
        if self.peek().type == type: return self.advance()

    def parseExpr(self):
        t = self.peek().type
        if 'yield' == t:
            self.advance()
            block = nodes.Block()
            block._yield = True
            return block
        elif t in ('id','class'):
            tok = self.advance()
            new_div = self.lexer.tok('tag','div')
            new_div.inline_level = tok.inline_level
            self.lexer.stash.append(new_div)
            self.lexer.stash.append(tok)
            return self.parseExpr()

        funcName = 'parse%s'%t.capitalize()
        if hasattr(self,funcName):
            return getattr(self,funcName)()
        else:
            raise Exception('unexpected token "%s" in file %s on line %d' %
                            (t, self.filename, self.line()))

    def parseString(self):
        tok = self.expect('string')
        node = nodes.String(tok.val, inline=tok.inline_level > 0)
        node.line = self.line()
        return node

    def parseText(self):
        tok = self.expect('text')
        node = nodes.Text(tok.val)
        node.line = self.line()
        return node

    def parseBlockExpansion(self):
        if ':'== self.peek().type:
            self.advance()
            return nodes.Block(self.parseExpr())
        else:
            return self.block()

    def parseAssignment(self):
        tok = self.expect('assignment')
        return nodes.Assignment(tok.name,tok.val)

    def parseCode(self):
        tok = self.expect('code')
        if not 'indent'==self.peek().type:
            node = nodes.Code(tok.val,tok.buffer,tok.escape) #tok.escape
            block,i = None,1
            node.line = self.line()
            while self.lookahead(i) and 'newline'==self.lookahead(i).type:
                i+= 1
            block = 'indent' == self.lookahead(i).type
            if block:
                self.skip(i-1)
                node.block = self.block()
        else:
            node = nodes.BlockCode(
                tok.val, self.parseTextBlock(), tok.buffer, tok.escape)
            node.line = self.line()
        return node

    def parseComment(self):
        tok = self.expect('comment')

        if 'indent'==self.peek().type:
            node = nodes.BlockComment(tok.val, self.block(), tok.buffer)
        else:
            node = nodes.Comment(tok.val,tok.buffer)

        node.line = self.line()
        return node

    def parseDoctype(self):
        tok = self.expect('doctype')
        node = nodes.Doctype(tok.val)
        node.line = self.line()
{'fraction_non_alphanumeric': 0.12590855803048065, 'fraction_numerical': 0.0018757327080890973, 'mean_word_length': 1.709114004445856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6766886', 'n_tokens_mistral': 2570, 'n_tokens_neox': 2474, 'n_words': 560}