<commit_before># ----------------------------------------------------------------------
# lexer.py
#
# Lexer for the Llama language
# http://courses.softlab.ntua.gr/compilers/2012a/llama2012.pdf
#
# Author: <NAME> <<EMAIL>>
#
# Lexer design is heavily inspired from the PHPLY lexer
# https://github.com/ramen/phply/blob/master/phply/phplex.py
# ----------------------------------------------------------------------

import ply.lex as lex

import error as err

# Represent reserved words as a frozenset for fast lookup
_reserved_words = frozenset('''
    and
    array
    begin
    bool
    char
    delete
    dim
    do
    done
    downto
    else
    end
    false
    float
    for
    if
    in
    int
    let
    match
    mod
    mutable
    new
    not
    of
    rec
    ref
    then
    to
    true
    type
    unit
    while
    with
    '''.split()
)

_reserved_tokens = tuple(s.upper() for s in _reserved_words)

_other_tokens = (
    # Identifiers (generic variable identifiers, constructor identifiers)
    'GENID', 'CONID',

    # Literals (int constant, float constant, char constant, string const)
    'ICONST', 'FCONST', 'CCONST', 'SCONST',

    # Integer operators (+, -, *, /)
    'PLUS', 'MINUS', 'TIMES', 'DIVIDE',

    # Floating-point operators (+., -., *., /., **)
    'FPLUS', 'FMINUS', 'FTIMES', 'FDIVIDE', 'FPOW',

    # Boolean operators (&&, ||)
    'BAND', 'BOR',

    # Comparison operators (<, <=, >, >=, =, <>, ==, !=)
    'LT', 'LE', 'GT', 'GE', 'EQ', 'NEQ', 'NATEQ', 'NATNEQ',

    # Pattern operators (->, |)
    'ARROW', 'PIPE',

    # Assignment and dereference (:=, !)
    'ASSIGN', 'BANG',

    # Semicolon (;)
    'SEMICOLON',

    # Delimeters ( ) [ ] , :
    'LPAREN', 'RPAREN',
    'LBRACKET', 'RBRACKET',
    'COMMA', 'COLON'
)

# All valid_tokens [exported]
tokens = _reserved_tokens + _other_tokens


class LlamaLexer:
    """An instrumented llama lexer"""

    # Token list needed by PLY
    tokens = tokens

    # Lexer states
    states = (
        ('comment', 'exclusive'),
        ('char',    'exclusive'),
        ('string',  'exclusive')
    )

    # The inner lexer, constructed by PLY.
    lexer = None

    # == REQUIRED ATTRIBUTES (export PLY interface) ==
    @property
    def lexdata(self):
        """The lexer's input string."""
        if self.lexer:
            return self.lexer.lexdata
        return None

    @lexdata.setter
    def lexdata(self, value):
        """Set lexer's input string."""
        if self.lexer:
            self.lexer.lexdata = value

    @property
    def lexmatch(self):
        """The latest matched token as a Match object."""
        if self.lexer:
            return self.lexer.lexmatch
        return None

    @property
    def lexpos(self):
        """Column following last token matched in current line."""
        if self.lexer:
            return self.lexer.lexpos - self.bol + 1
        return None

    # NOTE: There is no meaningful lexpos.setter method to define.
    @lexpos.setter
    def lexpos(self, value=None):
        raise NotImplementedError

    @property
    def lineno(self):
        """Current line of input"""
        if self.lexer:
            return self.lexer.lineno
        return None

    @lineno.setter
    def lineno(self, value):
        """Update line tracking"""
        if self.lexer:
            self.lexer.lineno = value

    # If debug is True, token will be duplicated to stdout.
    debug = False

    # File position of the most recent beginning of line
    bol = 0

    # Levels of nested comment blocks still open
    level = 0

    def __init__(self, debug=False):
        """
        Initialize wrapper object of PLY lexer. To get a working LlamaLexer,
        invoke build() on the returned object.
        """
        self.debug = debug

    # == ERROR PROCESSING ==

    # TODO: Make error style more gcc-like
    def error_out(self, message, lineno=None, lexpos=None):
        """Signal lexing error."""
        if lineno is not None:
            if lexpos is not None:
                s = "%d:%d Error: %s" % (lineno, lexpos, message)
            else:
                s = "%d: Error: %s" % (lineno, message)
        else:
            s = "Error: %s" % (message)
        err.push_error(lineno or 0, s)

    def warning_out(self, message, lineno=None, lexpos=None):
        """Signal lexing warning."""
        if lineno is not None:
            if lexpos is not None:
                s = "%s: %d:%d Warning: %s" % (lineno, lexpos, message)
            else:
                s = "%s: %d: Warning: %s" % (lineno, message)
        else:
            s = "%s: Warning: %s" % (message)
        err.push_warning(lineno or 0, s)

    # == REQUIRED LEXER INTERFACE ==

    def build(self, **kwargs):
        """
        Build a minimal lexer out of PLY and wrap it in a complete lexer
        for llama.

        NOTE: This function should always be called before ANY methods
        or attributes of the newly inidialized object are accessed.
        """
        self.lexer = lex.lex(module=self, **kwargs)
<commit_msg>Fix dosctring typos<commit_after># ----------------------------------------------------------------------
# lexer.py
#
# Lexer for the Llama language
# http://courses.softlab.ntua.gr/compilers/2012a/llama2012.pdf
#
# Author: <NAME> <<EMAIL>>
#
# Lexer design is heavily inspired from the PHPLY lexer
# https://github.com/ramen/phply/blob/master/phply/phplex.py
# ----------------------------------------------------------------------

import ply.lex as lex

import error as err

# Represent reserved words as a frozenset for fast lookup
_reserved_words = frozenset('''
    and
    array
    begin
    bool
    char
    delete
    dim
    do
    done
    downto
    else
    end
    false
    float
    for
    if
    in
    int
    let
    match
    mod
    mutable
    new
    not
    of
    rec
    ref
    then
    to
    true
    type
    unit
    while
    with
    '''.split()
)

_reserved_tokens = tuple(s.upper() for s in _reserved_words)

_other_tokens = (
    # Identifiers (generic variable identifiers, constructor identifiers)
    'GENID', 'CONID',

    # Literals (int constant, float constant, char constant, string const)
    'ICONST', 'FCONST', 'CCONST', 'SCONST',

    # Integer operators (+, -, *, /)
    'PLUS', 'MINUS', 'TIMES', 'DIVIDE',

    # Floating-point operators (+., -., *., /., **)
    'FPLUS', 'FMINUS', 'FTIMES', 'FDIVIDE', 'FPOW',

    # Boolean operators (&&, ||)
    'BAND', 'BOR',

    # Comparison operators (<, <=, >, >=, =, <>, ==, !=)
    'LT', 'LE', 'GT', 'GE', 'EQ', 'NEQ', 'NATEQ', 'NATNEQ',

    # Pattern operators (->, |)
    'ARROW', 'PIPE',

    # Assignment and dereference (:=, !)
    'ASSIGN', 'BANG',

    # Semicolon (;)
    'SEMICOLON',

    # Delimeters ( ) [ ] , :
    'LPAREN', 'RPAREN',
    'LBRACKET', 'RBRACKET',
    'COMMA', 'COLON'
)

# All valid_tokens [exported]
tokens = _reserved_tokens + _other_tokens


class LlamaLexer:
    """An instrumented llama lexer"""

    # Token list needed by PLY
    tokens = tokens

    # Lexer states
    states = (
        ('comment', 'exclusive'),
        ('char',    'exclusive'),
        ('string',  'exclusive')
    )

    # The inner lexer, constructed by PLY.
    lexer = None

    # == REQUIRED ATTRIBUTES (export PLY interface) ==
    @property
    def lexdata(self):
        """The lexer's input string."""
        if self.lexer:
            return self.lexer.lexdata
        return None

    @lexdata.setter
    def lexdata(self, value):
        """Set lexer's input string."""
        if self.lexer:
            self.lexer.lexdata = value

    @property
    def lexmatch(self):
        """The latest matched token as a Match object."""
        if self.lexer:
            return self.lexer.lexmatch
        return None

    @property
    def lexpos(self):
        """Column following last token matched in current line."""
        if self.lexer:
            return self.lexer.lexpos - self.bol + 1
        return None

    # NOTE: There is no meaningful lexpos.setter method to define.
    @lexpos.setter
    def lexpos(self, value=None):
        raise NotImplementedError

    @property
    def lineno(self):
        """Current line of input"""
        if self.lexer:
            return self.lexer.lineno
        return None

    @lineno.setter
    def lineno(self, value):
        """Update line tracking"""
        if self.lexer:
            self.lexer.lineno = value

    # If debug is True, token will be duplicated to stdout.
    debug = False

    # File position of the most recent beginning of line
    bol = 0

    # Levels of nested comment blocks still open
    level = 0

    def __init__(self, debug=False):
        """
        Initialize wrapper object of PLY lexer. To get a working LlamaLexer,
        invoke build() on the returned object.
        """
        self.debug = debug

    # == ERROR PROCESSING ==

    # TODO: Make error style more gcc-like
    def error_out(self, message, lineno=None, lexpos=None):
        """Signal lexing error."""
        if lineno is not None:
            if lexpos is not None:
                s = "%d:%d Error: %s" % (lineno, lexpos, message)
            else:
                s = "%d: Error: %s" % (lineno, message)
        else:
            s = "Error: %s" % (message)
        err.push_error(lineno or 0, s)

    def warning_out(self, message, lineno=None, lexpos=None):
        """Signal lexing warning."""
        if lineno is not None:
            if lexpos is not None:
                s = "%s: %d:%d Warning: %s" % (lineno, lexpos, message)
            else:
                s = "%s: %d: Warning: %s" % (lineno, message)
        else:
            s = "%s: Warning: %s" % (message)
        err.push_warning(lineno or 0, s)

    # == REQUIRED LEXER INTERFACE ==

    def build(self, **kwargs):
        """
        Build a lexer out of PLY and attach it to the wrapper object.

        NOTE: This function should be called once before ANY methods
        or attributes of the wrapper object are accessed.
        """
        self.lexer = lex.lex(module=self, **kwargs)

{'fraction_non_alphanumeric': 0.14806017418844022, 'fraction_numerical': 0.002573238321456849, 'mean_word_length': 1.8945860784875395, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3118804', 'n_tokens_mistral': 3313, 'n_tokens_neox': 3102, 'n_words': 958}