current position are we at eof reset position back up position by 1 return next token and advance position none if at eof todo return synthetic eof token instead of none return next token without advancing position return next token without advancing position if kind matches return next token and advance position if kind matches return next token and advance position requiring kind to match return source line lineno 1based construct a syntaxerror instance from message and token current position are we at eof reset position back up position by 1 return next token and advance position none if at eof todo return synthetic eof token instead of none return next token without advancing position return next token without advancing position if kind matches return next token and advance position if kind matches return next token and advance position requiring kind to match return source line lineno 1 based construct a syntaxerror instance from message and token
import lexer as lx

Token = lx.Token


class PLexer:
    def __init__(self, src: str, filename: str):
        self.src = src
        self.filename = filename
        self.tokens = list(lx.tokenize(self.src, filename=filename))
        self.pos = 0

    def getpos(self) -> int:
        
        return self.pos

    def eof(self) -> bool:
        
        return self.pos >= len(self.tokens)

    def setpos(self, pos: int) -> None:
        
        assert 0 <= pos <= len(self.tokens), (pos, len(self.tokens))
        self.pos = pos

    def backup(self) -> None:
        
        assert self.pos > 0
        self.pos -= 1

    def next(self, raw: bool = False) -> Token | None:
        
        
        while self.pos < len(self.tokens):
            tok = self.tokens[self.pos]
            self.pos += 1
            if raw or tok.kind != "COMMENT":
                return tok
        return None

    def peek(self, raw: bool = False) -> Token | None:
        
        tok = self.next(raw=raw)
        self.backup()
        return tok

    def maybe(self, kind: str, raw: bool = False) -> Token | None:
        
        tok = self.peek(raw=raw)
        if tok and tok.kind == kind:
            return tok
        return None

    def expect(self, kind: str) -> Token | None:
        
        tkn = self.next()
        if tkn is not None:
            if tkn.kind == kind:
                return tkn
            self.backup()
        return None

    def require(self, kind: str) -> Token:
        
        tkn = self.next()
        if tkn is not None and tkn.kind == kind:
            return tkn
        raise self.make_syntax_error(
            f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn
        )

    def extract_line(self, lineno: int) -> str:
        
        lines = self.src.splitlines()
        if lineno > len(lines):
            return ""
        return lines[lineno - 1]

    def make_syntax_error(self, message: str, tkn: Token | None = None) -> SyntaxError:
        
        if tkn is None:
            tkn = self.peek()
        if tkn is None:
            tkn = self.tokens[-1]
        return lx.make_syntax_error(
            message, self.filename, tkn.line, tkn.column, self.extract_line(tkn.line)
        )


if __name__ == "__main__":
    import sys

    if sys.argv[1:]:
        filename = sys.argv[1]
        if filename == "-c" and sys.argv[2:]:
            src = sys.argv[2]
            filename = "<string>"
        else:
            with open(filename) as f:
                src = f.read()
    else:
        filename = "<default>"
        src = "if (x) { x.foo; // comment\n}"
    p = PLexer(src, filename)
    while not p.eof():
        tok = p.next(raw=True)
        assert tok
        left = repr(tok)
        right = lx.to_text([tok]).rstrip()
        print(f"{left:40.40} {right}")