<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers.csound
    ~~~~~~~~~~~~~~~~~~~~~~

    Lexers for Csound languages.

    :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, bygroups, default, include, using, words
from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \
    String, Text
from pygments.lexers._csound_builtins import OPCODES
from pygments.lexers.html import HtmlLexer
from pygments.lexers.python import PythonLexer
from pygments.lexers.scripting import LuaLexer

__all__ = ['CsoundScoreLexer', 'CsoundOrchestraLexer', 'CsoundDocumentLexer']

newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text))


class CsoundLexer(RegexLexer):
    tokens = {
        'whitespace': [
            (r'[ \t]+', Text),
            (r'\\\n', Text),
            (r'/[*](.|\n)*?[*]/', Comment.Multiline)
        ],

        'macro use': [
            (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation),
             'function macro use'),
            (r'\$\w+(\.|\b)', Comment.Preproc)
        ],
        'function macro use': [
            (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)),
            (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop')
        ],

        'whitespace or macro use': [
            include('whitespace'),
            include('macro use')
        ],

        'preprocessor directives': [
            (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc),
            (r'#include\b', Comment.Preproc, 'include'),
            (r'#[ \t]*define\b', Comment.Preproc, 'macro name'),
            (r'@+[ \t]*\d*', Comment.Preproc)
        ],

        'include': [
            include('whitespace'),
            (r'([^ \t]).*?\1', String, '#pop')
        ],

        'macro name': [
            include('whitespace'),
            (r'(\w+)(\()', bygroups(Comment.Preproc, Text),
             'function macro parameter list'),
            (r'\w+', Comment.Preproc, 'object macro definition after name')
        ],
        'object macro definition after name': [
            include('whitespace'),
            (r'#', Punctuation, 'object macro replacement text')
        ],
        'object macro replacement text': [
            (r'(\\#|[^#])+', Comment.Preproc),
            (r'#', Punctuation, '#pop:3')
        ],
        'function macro parameter list': [
            (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)),
            (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation),
             'function macro definition after parameter list')
        ],
        'function macro definition after parameter list': [
            (r'[ \t]+', Text),
            (r'#', Punctuation, 'function macro replacement text')
        ],
        'function macro replacement text': [
            (r'(\\#|[^#])+', Comment.Preproc),
            (r'#', Punctuation, '#pop:4')
        ]
    }


class CsoundScoreLexer(CsoundLexer):
    """
    For `Csound <http://csound.github.io>`_ scores.

    .. versionadded:: 2.1
    """

    name = 'Csound Score'
    aliases = ['csound-score', 'csound-sco']
    filenames = ['*.sco']

    tokens = {
        'partial statement': [
            include('preprocessor directives'),
            (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
            (r'0[xX][a-fA-F0-9]+', Number.Hex),
            (r'\d+', Number.Integer),<commit_msg>Match Csound preprocessor<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers.csound
    ~~~~~~~~~~~~~~~~~~~~~~

    Lexers for Csound languages.

    :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, bygroups, default, include, using, words
from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \
    String, Text
from pygments.lexers._csound_builtins import OPCODES
from pygments.lexers.html import HtmlLexer
from pygments.lexers.python import PythonLexer
from pygments.lexers.scripting import LuaLexer

__all__ = ['CsoundScoreLexer', 'CsoundOrchestraLexer', 'CsoundDocumentLexer']

newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text))


class CsoundLexer(RegexLexer):
    tokens = {
        'whitespace': [
            (r'[ \t]+', Text),
            (r'\\\n', Text),
            (r'/[*](.|\n)*?[*]/', Comment.Multiline)
        ],

        'macro use': [
            (r'(\$[A-Z_a-z]\w*\.?)(\()', bygroups(Comment.Preproc, Punctuation),
             'function macro use'),
            (r'\$[A-Z_a-z]\w*(\.|\b)', Comment.Preproc)
        ],
        'function macro use': [
            (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)),
            (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop')
        ],

        'whitespace or macro use': [
            include('whitespace'),
            include('macro use')
        ],

        'preprocessor directives': [
            (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc),
            (r'#include', Comment.Preproc, 'include'),
            (r'#[ \t]*define', Comment.Preproc, 'macro name'),
            (r'@+[ \t]*\d*', Comment.Preproc)
        ],

        'include': [
            include('whitespace'),
            (r'([^ \t]).*?\1', String, '#pop')
        ],

        'macro name': [
            include('whitespace'),
            (r'([A-Z_a-z]\w*)(\()', bygroups(Comment.Preproc, Text),
             'function macro parameter list'),
            (r'[A-Z_a-z]\w*', Comment.Preproc, 'object macro definition after name')
        ],
        'object macro definition after name': [
            include('whitespace'),
            (r'#', Punctuation, 'object macro replacement text')
        ],
        'object macro replacement text': [
            (r'(\\#|[^#])+', Comment.Preproc),
            (r'#', Punctuation, '#pop:3')
        ],
        'function macro parameter list': [
            include('whitespace'),
            (r"([A-Z_a-z]\w*)(['#])", bygroups(Comment.Preproc, Punctuation)),
            (r'([A-Z_a-z]\w*)(\))', bygroups(Comment.Preproc, Punctuation),
             'function macro definition after parameter list')
        ],
        'function macro definition after parameter list': [
            (r'[ \t]+', Text),
            (r'#', Punctuation, 'function macro replacement text')
        ],
        'function macro replacement text': [
            (r'(\\#|[^#])+', Comment.Preproc),
            (r'#', Punctuation, '#pop:4')
        ]
    }


class CsoundScoreLexer(CsoundLexer):
    """
    For `Csound <http://csound.github.io>`_ scores.

    .. versionadded:: 2.1
    """

    name = 'Csound Score'
    aliases = ['csound-score', 'csound-sco']
    filenames = ['*.sco']

    tokens = {
        'partial statement': [
            include('preprocessor directives'),
            (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
            (r'0[xX][a-fA-F0-9]+', Number.Hex),
            (r'\d+', Number.Integer),
{'fraction_non_alphanumeric': 0.20582062403805793, 'fraction_numerical': 0.004757240800335805, 'mean_word_length': 2.2169216921692168, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '741217', 'n_tokens_mistral': 2415, 'n_tokens_neox': 2289, 'n_words': 468}