<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers
    ~~~~~~~~~~~~~~~

    Pygments lexers.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys
import types
import fnmatch
import re
from os.path import basename

from pygments.lexers._mapping import LEXERS
from pygments.modeline import get_filetype_from_buffer
from pygments.plugin import find_plugin_lexers
from pygments.util import ClassNotFound, itervalues


__all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
           'guess_lexer'] + list(LEXERS)

_lexer_cache = {}
_pattern_cache = {}

def _fn_matches(fn, glob):
    """
    Return whether the supplied file name fn matches pattern filename
    """
    if glob not in _pattern_cache:
        pattern = re.compile(fnmatch.translate(glob))
        _pattern_cache[glob] = pattern
    else:
        pattern = _pattern_cache[glob]

    return pattern.match(fn)


def _load_lexers(module_name):
    """
    Load a lexer (and all others in the module too).
    """
    mod = __import__(module_name, None, None, ['__all__'])
    for lexer_name in mod.__all__:
        cls = getattr(mod, lexer_name)
        _lexer_cache[cls.name] = cls


def get_all_lexers():
    """
    Return a generator of tuples in the form ``(name, aliases,
    filenames, mimetypes)`` of all know lexers.
    """
    for item in itervalues(LEXERS):
        yield item[1:]
    for lexer in find_plugin_lexers():
        yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes


def find_lexer_class(name):
    """
    Lookup a lexer class by name. Return None if not found.
    """
    if name in _lexer_cache:
        return _lexer_cache[name]
    # lookup builtin lexers
    for module_name, lname, aliases, _, _ in itervalues(LEXERS):
        if name == lname:
            _load_lexers(module_name)
            return _lexer_cache[name]
    # continue with lexers from setuptools entrypoints
    for cls in find_plugin_lexers():
        if cls.name == name:
            return cls


def get_lexer_by_name(_alias, **options):
    """
    Get a lexer by an alias.
    """
    # lookup builtin lexers
    for module_name, name, aliases, _, _ in itervalues(LEXERS):
        if _alias.lower() in aliases:
            if name not in _lexer_cache:
                _load_lexers(module_name)
            return _lexer_cache[name](**options)
    # continue with lexers from setuptools entrypoints
    for cls in find_plugin_lexers():
        if _alias in cls.aliases:
            return cls(**options)
    raise ClassNotFound('no lexer for alias %r found' % _alias)


def get_lexer_for_filename(_fn, code=None, **options):
    """
    Get a lexer for a filename.  If multiple lexers match the filename
    pattern, use ``analyze_text()`` to figure out which one is more
    appropriate.
    """
    matches = []
    fn = basename(_fn)
    for modname, name, _, filenames, _ in itervalues(LEXERS):
        for filename in filenames:
            if _fn_matches(fn, filename):
                if name not in _lexer_cache:
                    _load_lexers(modname)
                matches.append((_lexer_cache[name], filename))
    for cls in find_plugin_lexers():<commit_msg>Fix docstring from analyze_text to analyse_text<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers
    ~~~~~~~~~~~~~~~

    Pygments lexers.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys
import types
import fnmatch
import re
from os.path import basename

from pygments.lexers._mapping import LEXERS
from pygments.modeline import get_filetype_from_buffer
from pygments.plugin import find_plugin_lexers
from pygments.util import ClassNotFound, itervalues


__all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
           'guess_lexer'] + list(LEXERS)

_lexer_cache = {}
_pattern_cache = {}

def _fn_matches(fn, glob):
    """
    Return whether the supplied file name fn matches pattern filename
    """
    if glob not in _pattern_cache:
        pattern = re.compile(fnmatch.translate(glob))
        _pattern_cache[glob] = pattern
    else:
        pattern = _pattern_cache[glob]

    return pattern.match(fn)


def _load_lexers(module_name):
    """
    Load a lexer (and all others in the module too).
    """
    mod = __import__(module_name, None, None, ['__all__'])
    for lexer_name in mod.__all__:
        cls = getattr(mod, lexer_name)
        _lexer_cache[cls.name] = cls


def get_all_lexers():
    """
    Return a generator of tuples in the form ``(name, aliases,
    filenames, mimetypes)`` of all know lexers.
    """
    for item in itervalues(LEXERS):
        yield item[1:]
    for lexer in find_plugin_lexers():
        yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes


def find_lexer_class(name):
    """
    Lookup a lexer class by name. Return None if not found.
    """
    if name in _lexer_cache:
        return _lexer_cache[name]
    # lookup builtin lexers
    for module_name, lname, aliases, _, _ in itervalues(LEXERS):
        if name == lname:
            _load_lexers(module_name)
            return _lexer_cache[name]
    # continue with lexers from setuptools entrypoints
    for cls in find_plugin_lexers():
        if cls.name == name:
            return cls


def get_lexer_by_name(_alias, **options):
    """
    Get a lexer by an alias.
    """
    # lookup builtin lexers
    for module_name, name, aliases, _, _ in itervalues(LEXERS):
        if _alias.lower() in aliases:
            if name not in _lexer_cache:
                _load_lexers(module_name)
            return _lexer_cache[name](**options)
    # continue with lexers from setuptools entrypoints
    for cls in find_plugin_lexers():
        if _alias in cls.aliases:
            return cls(**options)
    raise ClassNotFound('no lexer for alias %r found' % _alias)


def get_lexer_for_filename(_fn, code=None, **options):
    """
    Get a lexer for a filename.  If multiple lexers match the filename
    pattern, use ``analyse_text()`` to figure out which one is more
    appropriate.
    """
    matches = []
    fn = basename(_fn)
    for modname, name, _, filenames, _ in itervalues(LEXERS):
        for filename in filenames:
            if _fn_matches(fn, filename):
                if name not in _lexer_cache:
                    _load_lexers(modname)
                matches.append((_lexer_cache[name], filename))
    for cls in find_plugin_lexers():
{'fraction_non_alphanumeric': 0.09467275494672756, 'fraction_numerical': 0.0030441400304414, 'mean_word_length': 2.8203488372093024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4524689', 'n_tokens_mistral': 2164, 'n_tokens_neox': 2072, 'n_words': 608}