<commit_before>#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: Apache 2.0 Copyright: 2017, <NAME> <<EMAIL>id at kovidgoyal.net>

from __future__ import absolute_import, division, print_function, unicode_literals

import codecs
import importlib
import sys
from collections import namedtuple
from locale import getpreferredencoding

if not hasattr(sys, 'generating_docs_via_sphinx'):
    from lxml import etree  # Must be imported before html_parser to initialize libxml

    try:
        from . import html_parser
    except ImportError:
        raise
    else:
        version = namedtuple('Version', 'major minor patch')(
            html_parser.MAJOR, html_parser.MINOR, html_parser.PATCH)

        if not hasattr(etree, 'adopt_external_document'):
            raise ImportError('Your version of lxml is too old, version 3.8.0 is minimum')

        LIBXML_VERSION = ((html_parser.LIBXML_VERSION // 10000) % 100,
                          (html_parser.LIBXML_VERSION // 100) % 100,
                          html_parser.LIBXML_VERSION % 100, )
        if LIBXML_VERSION != etree.LIBXML_VERSION:
            raise RuntimeError(
                'html5-parser and lxml are using different versions of libxml2.'
                ' This happens commonly when using pip installed versions of lxml.'
                ' Use pip install --no-binary lxml lxml instead.'
                ' libxml2 versions: html5-parser: {} != lxml: {}'.format(
                    LIBXML_VERSION, etree.LIBXML_VERSION))

BOMS = (codecs.BOM_UTF8, codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)


def check_bom(data):
    for bom in BOMS:
        if data.startswith(bom):
            return bom


def check_for_meta_charset(raw):
    from .encoding_parser import EncodingParser  # delay load
    q = raw[:10 * 1024]
    parser = EncodingParser(q)
    encoding = parser()
    if encoding in ("utf-16", "utf-16be", "utf-16le"):
        encoding = "utf-8"
    return encoding


def detect_encoding(raw):
    from chardet import detect  # delay load<commit_msg>Only check the major and minor versions of libxml<commit_after>#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: Apache 2.0 Copyright: 2017, <NAME> <kovid at kovidgoyal.net>

from __future__ import absolute_import, division, print_function, unicode_literals

import codecs
import importlib
import sys
from collections import namedtuple
from locale import getpreferredencoding

if not hasattr(sys, 'generating_docs_via_sphinx'):
    from lxml import etree  # Must be imported before html_parser to initialize libxml

    try:
        from . import html_parser
    except ImportError:
        raise
    else:
        version = namedtuple('Version', 'major minor patch')(
            html_parser.MAJOR, html_parser.MINOR, html_parser.PATCH)

        if not hasattr(etree, 'adopt_external_document'):
            raise ImportError('Your version of lxml is too old, version 3.8.0 is minimum')

        LIBXML_VERSION = ((html_parser.LIBXML_VERSION // 10000) % 100,
                          (html_parser.LIBXML_VERSION // 100) % 100,
                          html_parser.LIBXML_VERSION % 100, )
        if LIBXML_VERSION[:2] != etree.LIBXML_VERSION[:2]:
            raise RuntimeError(
                'html5-parser and lxml are using different versions of libxml2.'
                ' This happens commonly when using pip installed versions of lxml.'
                ' Use pip install --no-binary lxml lxml instead.'
                ' libxml2 versions: html5-parser: {} != lxml: {}'.format(
                    LIBXML_VERSION, etree.LIBXML_VERSION))

BOMS = (codecs.BOM_UTF8, codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)


def check_bom(data):
    for bom in BOMS:
        if data.startswith(bom):
            return bom


def check_for_meta_charset(raw):
    from .encoding_parser import EncodingParser  # delay load
    q = raw[:10 * 1024]
    parser = EncodingParser(q)
    encoding = parser()
    if encoding in ("utf-16", "utf-16be", "utf-16le"):
        encoding = "utf-8"
    return encoding


def detect_encoding(raw):
    from chardet import detect  # delay load
{'fraction_non_alphanumeric': 0.08674463937621832, 'fraction_numerical': 0.024366471734892786, 'mean_word_length': 2.694869486948695, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 21}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5690734', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1210, 'n_words': 357}