<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers.graph
    ~~~~~~~~~~~~~~~~~~~~~

    Lexers for graph query languages.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, include, bygroups
from pygments.token import Keyword, Punctuation, Text, Comment, Operator, Name,\
String, Number, Generic


__all__ = ['CypherLexer']


class CypherLexer(RegexLexer):
    """
    For `Cypher Query Language
    <http://docs.neo4j.org/chunked/milestone/cypher-query-lang.html>`_

    For the Cypher version in Neo4J 2.0

    .. versionadded:: 2.0
    """
    name = 'Cypher'
    aliases = ['cypher']
    filenames = ['*.cyp','*.cypher']
    flags = re.MULTILINE | re.IGNORECASE

    tokens = {
        'root': [
            include('comment'),
            include('keywords'),
            include('clauses'),
            include('relations'),
            include('strings')
            ],
        'comment': [(r'^.*//.*\n', Comment.Single)],
        'keywords': [
            (r'create|order|match|limit|set|skip|start|return|with|where|delete'
             r'|foreach|not| by ', Keyword)],
        'clauses': [(r' all | any | as | asc |create|create unique|delete|'
                     r'desc |distinct|foreach| in |is null|limit|match|none|'
                     r'order by|return|set|skip|single|start|union|where|with',
                     Keyword)],
        'relations': [(r'-->|-\[.*\]->|<-\[.*\]-|<--|\[|\]', Operator),
                      (r'<|>|<>|=|<=|=>|\(|\)|\||:|,|;', Punctuation)],
        'strings': [(r'\".*\"', String)]
        }


<commit_msg>Reflow Cypher lexer<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers.graph
    ~~~~~~~~~~~~~~~~~~~~~

    Lexers for graph query languages.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, include, bygroups
from pygments.token import Keyword, Punctuation, Text, Comment, Operator, Name,\
String, Number, Generic


__all__ = ['CypherLexer']


class CypherLexer(RegexLexer):
    """
    For `Cypher Query Language
    <http://docs.neo4j.org/chunked/milestone/cypher-query-lang.html>`_

    For the Cypher version in Neo4J 2.0

    .. versionadded:: 2.0
    """
    name = 'Cypher'
    aliases = ['cypher']
    filenames = ['*.cyp','*.cypher']
    flags = re.MULTILINE | re.IGNORECASE

    tokens = {
        'root': [
            include('comment'),
            include('keywords'),
            include('clauses'),
            include('relations'),
            include('strings')
            ],
        'comment': [
            (r'^.*//.*\n', Comment.Single),
        ],
        'keywords': [
            (r'create|order|match|limit|set|skip|start|return|with|where|delete'
             r'|foreach|not| by ', Keyword),
        ],
        'clauses': [
            (r' all | any | as | asc |create|create unique|delete|'
             r'desc |distinct|foreach| in |is null|limit|match|none|'
             r'order by|return|set|skip|single|start|union|where|with',
             Keyword),
        ],
        'relations': [
            (r'-->|-\[.*\]->|<-\[.*\]-|<--|\[|\]', Operator),
            (r'<|>|<>|=|<=|=>|\(|\)|\||:|,|;', Punctuation),
        ],
        'strings': [
            (r'\".*\"', String),
        ]
    }



{'fraction_non_alphanumeric': 0.19610691458454388, 'fraction_numerical': 0.008715862870424172, 'mean_word_length': 2.187962962962963, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2453539', 'n_tokens_mistral': 1197, 'n_tokens_neox': 1089, 'n_words': 222}