<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers.puppet
    ~~~~~~~~~~~~~~~~~~~~~~

    Lexer for the Puppet DSL.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from pygments.lexer import RegexLexer, bygroups, include
from pygments.token import Comment, Keyword, Name, Number, Operator
from pygments.token import Punctuation, String, Text


__all__ = ['PuppetLexer']


class PuppetLexer(RegexLexer):
    name = 'Puppet'
    aliases = ['puppet']
    filenames = ['*.pp']

    tokens = {
        'root': [
            include('comments'),
            include('keywords'),
            include('names'),
            include('numbers'),
            include('operators'),
            include('strings'),

            (r'[]{}:(),;[]', Punctuation),
            (r'[^\S\n]+', Text),
        ],

        'comments': [
            (r'\s*#.*$', Comment),
            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
        ],

        'operators': [
            (r'(=>|\?|<|>|=|\+|-|\/|\*|~|!|\|)', Operator),
            (r'\s+(in|and|or|not)\s+', Operator.Word),
        ],

        'names': [
            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute),

            # FIXME $variable[array] doesn't look right
            (r'\$\S+', Name.Variable),
        ],

        'numbers': [
            # Copypasta from the Python lexer
            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
            (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
            (r'0[0-7]+j?', Number.Oct),
            (r'0[xX][a-fA-F0-9]+', Number.Hex),
            (r'\d+L', Number.Integer.Long),
            (r'\d+j?', Number.Integer)
        ],

        'keywords': [
            # Left out 'group' and 'require' as they are used as attributes
            (r'(?i)(absent|alert|alias|audit|augeas|before|case)\b', Keyword),
            (r'(?i)(check|class|computer|configured|contained)\b', Keyword),
            (r'(?i)(create_resources|crit|cron|debug|default)\b', Keyword),
            (r'(?i)(define|defined|directory|else|elsif|emerg)\b', Keyword),
            (r'(?i)(err|exec|extlookup|fail|false|file)\b', Keyword),
            (r'(?i)(filebucket|fqdn_rand|generate|host|if|import)\b', Keyword),
            (r'(?i)(include|info|inherits|inline_template)\b', Keyword),
            (r'(?i)(installed|interface|k5login|latest|link)\b', Keyword),
            (r'(?i)(loglevel|macauthorization|mailalias|maillist)\b', Keyword),
            (r'(?i)(mcx|md5|mount|mounted|nagios_command)\b', Keyword),
            (r'(?i)(nagios_contact|nagios_contactgroup)\b', Keyword),
            (r'(?i)(nagios_host|nagios_hostdependency)\b', Keyword),
            (r'(?i)(nagios_hostescalation|nagios_hostextinfo)\b', Keyword),
            (r'(?i)(nagios_hostgroup|nagios_service)\b', Keyword),
            (r'(?i)(nagios_servicedependency)\b', Keyword),
            (r'(?i)(nagios_serviceescalation)\b', Keyword),
            (r'(?i)(nagios_serviceextinfo|nagios_servicegroup)\b', Keyword),
            (r'(?i)(nagios_timeperiod|node|noop|notice|notify)\b', Keyword),
            (r'(?i)(package|present|purged|realize|regsubst)\b', Keyword),
            (r'(?i)(resources|role|router|running)\b', Keyword),
            (r'(?i)(schedule|scheduled_task|search|selboolean)\b', Keyword),
            (r'(?i)(selmodule|service|sha1|shellquote|split)\b', Keyword),
            (r'(?i)(sprintf|ssh_authorized_key|sshkey|stage)\b', Keyword),
            (r'(?i)(stopped|subscribe|tag|tagged|template|tidy)\b', Keyword),
            (r'(?i)(true|undef|unmounted|user|versioncmp|vlan)\b', Keyword),
            (r'(?i)(warning|yumrepo|zfs|zone|zpool)\b', Keyword),
        ],

        'strings': [
            (r'["\'](.*?)["\']', String),
            (r'["\'](.*\n.*)*["\']', String),
        ],

    }
<commit_msg>better handling of multiline strings<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers.puppet
    ~~~~~~~~~~~~~~~~~~~~~~

    Lexer for the Puppet DSL.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from pygments.lexer import RegexLexer, bygroups, include
from pygments.token import Comment, Keyword, Name, Number, Operator
from pygments.token import Punctuation, String, Text


__all__ = ['PuppetLexer']


class PuppetLexer(RegexLexer):
    name = 'Puppet'
    aliases = ['puppet']
    filenames = ['*.pp']

    tokens = {
        'root': [
            include('comments'),
            include('keywords'),
            include('names'),
            include('numbers'),
            include('operators'),
            include('strings'),

            (r'[]{}:(),;[]', Punctuation),
            (r'[^\S\n]+', Text),
        ],

        'comments': [
            (r'\s*#.*$', Comment),
            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
        ],

        'operators': [
            (r'(=>|\?|<|>|=|\+|-|\/|\*|~|!|\|)', Operator),
            (r'\s+(in|and|or|not)\s+', Operator.Word),
        ],

        'names': [
            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute),

            # FIXME $variable[array] doesn't look right
            (r'\$\S+', Name.Variable),
        ],

        'numbers': [
            # Copypasta from the Python lexer
            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
            (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
            (r'0[0-7]+j?', Number.Oct),
            (r'0[xX][a-fA-F0-9]+', Number.Hex),
            (r'\d+L', Number.Integer.Long),
            (r'\d+j?', Number.Integer)
        ],

        'keywords': [
            # Left out 'group' and 'require' as they are used as attributes
            (r'(?i)(absent|alert|alias|audit|augeas|before|case)\b', Keyword),
            (r'(?i)(check|class|computer|configured|contained)\b', Keyword),
            (r'(?i)(create_resources|crit|cron|debug|default)\b', Keyword),
            (r'(?i)(define|defined|directory|else|elsif|emerg)\b', Keyword),
            (r'(?i)(err|exec|extlookup|fail|false|file)\b', Keyword),
            (r'(?i)(filebucket|fqdn_rand|generate|host|if|import)\b', Keyword),
            (r'(?i)(include|info|inherits|inline_template)\b', Keyword),
            (r'(?i)(installed|interface|k5login|latest|link)\b', Keyword),
            (r'(?i)(loglevel|macauthorization|mailalias|maillist)\b', Keyword),
            (r'(?i)(mcx|md5|mount|mounted|nagios_command)\b', Keyword),
            (r'(?i)(nagios_contact|nagios_contactgroup)\b', Keyword),
            (r'(?i)(nagios_host|nagios_hostdependency)\b', Keyword),
            (r'(?i)(nagios_hostescalation|nagios_hostextinfo)\b', Keyword),
            (r'(?i)(nagios_hostgroup|nagios_service)\b', Keyword),
            (r'(?i)(nagios_servicedependency)\b', Keyword),
            (r'(?i)(nagios_serviceescalation)\b', Keyword),
            (r'(?i)(nagios_serviceextinfo|nagios_servicegroup)\b', Keyword),
            (r'(?i)(nagios_timeperiod|node|noop|notice|notify)\b', Keyword),
            (r'(?i)(package|present|purged|realize|regsubst)\b', Keyword),
            (r'(?i)(resources|role|router|running)\b', Keyword),
            (r'(?i)(schedule|scheduled_task|search|selboolean)\b', Keyword),
            (r'(?i)(selmodule|service|sha1|shellquote|split)\b', Keyword),
            (r'(?i)(sprintf|ssh_authorized_key|sshkey|stage)\b', Keyword),
            (r'(?i)(stopped|subscribe|tag|tagged|template|tidy)\b', Keyword),
            (r'(?i)(true|undef|unmounted|user|versioncmp|vlan)\b', Keyword),
            (r'(?i)(warning|yumrepo|zfs|zone|zpool)\b', Keyword),
        ],

        'strings': [
            (r'"([^"])*"', String),
            (r'\'([^\'])*\'', String),
        ],

    }

{'fraction_non_alphanumeric': 0.22336858395666753, 'fraction_numerical': 0.006190353366004642, 'mean_word_length': 2.5508241758241756, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1276868', 'n_tokens_mistral': 2823, 'n_tokens_neox': 2694, 'n_words': 350}