<commit_before># -*- coding: utf-8 -*-

"""Pygments lexer for SISAL.

SISAL stands for Streams and Iteration In A Single-Assignment Language.

"""


from pygments.lexer import RegexLexer, Text, words, bygroups, include, default
from pygments.token import Comment, Name, Keyword, Number, Operator, String

__version__ = '0.1.2'
__author__ = '<NAME>'
__license__ = 'MIT'
__all__ = ['SisalLexer']


class SisalLexer(RegexLexer):
    """Lexer for the SISAL programming language."""

    name = 'sisal'
    aliases = ['sis', 'sisal']
    filenames = ['*.sis', '*.sisal']
    tokens = {
        'root': [
            (r'\n', Text),
            (r'\\\n', Text),
            (r'\\', Text),
            (r'(define)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'(global)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'^%\$', Comment.Directive, 'directive'),
            (r'% .+$', Comment.Singleline),
            (r'\'', String.Single, 'singlequoted_string'),
            (r'"', String.Double, 'doublequoted_string'),
            (r'(function)(\s+)(.+)(\()',
             bygroups(Keyword, Text, Name.Function, Text)),
            (r'error', Name.Exception),
            (r'\(|\)', Text),
            include('keywords'),
            include('conversions'),
            include('types'),
            include('builtins'),
            include('numbers'),
            include('operators'),
            include('conditionals'),
            include('loops'),
            (r'\w+', Text),
            (r'[^\S\n]+', Text),
        ],
        'keywords': [
            (words(['sum',
                    'product',
                    'greatest',
                    'least',
                    'catenate',
                    'let',
                    'is',
                    'tagcase',
                    'otherwise',
                    'type',
                    'returns',
                    'forward',
                    # 'define',
                    'old',
                    'of',
                    'nil',
                    'in',
                    'end let',
                    'end function'],
                   prefix=r'\b',
                   suffix=r'\b'),
             Keyword.Reserved)
        ],
        'types': [<commit_msg>Made highlighting of 'of' syntax uniform<commit_after># -*- coding: utf-8 -*-

"""Pygments lexer for SISAL.

SISAL stands for Streams and Iteration In A Single-Assignment Language.

"""


from pygments.lexer import RegexLexer, Text, words, bygroups, include, default
from pygments.token import Comment, Name, Keyword, Number, Operator, String

__version__ = '0.1.2'
__author__ = '<NAME>'
__license__ = 'MIT'
__all__ = ['SisalLexer']


class SisalLexer(RegexLexer):
    """Lexer for the SISAL programming language."""

    name = 'sisal'
    aliases = ['sis', 'sisal']
    filenames = ['*.sis', '*.sisal']
    tokens = {
        'root': [
            (r'\n', Text),
            (r'\\\n', Text),
            (r'\\', Text),
            (r'(define)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'(global)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'^%\$', Comment.Directive, 'directive'),
            (r'% .+$', Comment.Singleline),
            (r'\'', String.Single, 'singlequoted_string'),
            (r'"', String.Double, 'doublequoted_string'),
            (r'(function)(\s+)(.+)(\()',
             bygroups(Keyword, Text, Name.Function, Text)),
            (r'error', Name.Exception),
            (r'\(|\)', Text),
            include('keywords'),
            include('conversions'),
            include('types'),
            include('builtins'),
            include('numbers'),
            include('operators'),
            include('conditionals'),
            include('loops'),
            (r'\w+', Text),
            (r'[^\S\n]+', Text),
        ],
        'keywords': [
            (words(['sum',
                    'product',
                    'greatest',
                    'least',
                    'catenate',
                    'let',
                    'is',
                    'tagcase',
                    'otherwise',
                    'type',
                    'returns',
                    'forward',
                    'array of',
                    'stream of',
                    'value of',
                    'old',
                    'of',
                    'nil',
                    'in',
                    'end let',
                    'end function'],
                   prefix=r'\b',
                   suffix=r'\b'),
             Keyword.Reserved)
        ],
        'types': [
{'fraction_non_alphanumeric': 0.17125053717232488, 'fraction_numerical': 0.0017189514396218307, 'mean_word_length': 1.3462701612903225, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3278116', 'n_tokens_mistral': 1366, 'n_tokens_neox': 1277, 'n_words': 276}