<commit_before>import unittest, codecs
import six
from w3lib.encoding import (html_body_declared_encoding, read_bom, to_unicode,
        http_content_type_encoding, resolve_encoding, html_to_unicode)

class RequestEncodingTests(unittest.TestCase):
    utf8_fragments = [
        # Content-Type as meta http-equiv
        b"""<meta http-equiv="content-type" content="text/html;charset=UTF-8" />""",
        b"""\n<meta http-equiv="Content-Type"\ncontent="text/html; charset=utf-8">""",
        b"""<meta content="text/html; charset=utf-8"\n http-equiv='Content-Type'>""",
        b""" bad html still supported < meta http-equiv='Content-Type'\n content="text/html; charset=utf-8">""",
        # html5 meta charset
        b"""<meta charset="utf-8">""",
        # xml encoding
        b"""<?xml version="1.0" encoding="utf-8"?>""",
    ]

    def test_bom(self):
        # cjk water character in unicode<commit_msg>add test for encoding as a separate attribute<commit_after>import unittest, codecs
import six
from w3lib.encoding import (html_body_declared_encoding, read_bom, to_unicode,
        http_content_type_encoding, resolve_encoding, html_to_unicode)

class RequestEncodingTests(unittest.TestCase):
    utf8_fragments = [
        # Content-Type as meta http-equiv
        b"""<meta http-equiv="content-type" content="text/html;charset=UTF-8" />""",
        b"""\n<meta http-equiv="Content-Type"\ncontent="text/html; charset=utf-8">""",
        b"""<meta http-equiv="Content-Type" content="text/html" charset="utf-8">""",
        b"""<meta content="text/html; charset=utf-8"\n http-equiv='Content-Type'>""",
        b""" bad html still supported < meta http-equiv='Content-Type'\n content="text/html; charset=utf-8">""",
        # html5 meta charset
        b"""<meta charset="utf-8">""",
        # xml encoding
        b"""<?xml version="1.0" encoding="utf-8"?>""",
    ]

    def test_bom(self):
        # cjk water character in unicode
{'fraction_non_alphanumeric': 0.1646937725167267, 'fraction_numerical': 0.011837364899639732, 'mean_word_length': 3.6956521739130435, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 2, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5405617', 'n_tokens_mistral': 631, 'n_tokens_neox': 607, 'n_words': 124}