<commit_before>import io
import os.path
import nose.tools as nt

from IPython.utils import openpy

mydir = os.path.dirname(__file__)
nonascii_path = os.path.join(mydir, '../../core/tests/nonascii.py')

def test_detect_encoding():
    with open(nonascii_path, 'rb') as f:
        enc, lines = openpy.detect_encoding(f.readline)
    nt.assert_equal(enc, 'iso-8859-5')

def test_read_file():
    with io.open(nonascii_path, encoding='iso-8859-5') as f:
        read_specified_enc = f.read()
    read_detected_enc = openpy.read_py_file(nonascii_path, skip_encoding_cookie=False)
    nt.assert_equal(read_detected_enc, read_specified_enc)
    assert 'coding: iso-8859-5' in read_detected_enc
    
    read_strip_enc_cookie = openpy.read_py_file(nonascii_path, skip_encoding_cookie=True)
    assert 'coding: iso-8859-5' not in read_strip_enc_cookie

def test_source_to_unicode():
    with io.open(nonascii_path, 'rb') as f:
        source_bytes = f.read()
    nt.assert_equal(openpy.source_to_unicode(source_bytes, skip_encoding_cookie=False).splitlines(),
                    source_bytes.decode('iso-8859-5').splitlines())

    source_no_cookie = openpy.source_to_unicode(source_bytes, skip_encoding_cookie=True)
    nt.assert_not_in('coding: iso-8859-5', source_no_cookie)<commit_msg>apply black on the file<commit_after>import io
import os.path
import nose.tools as nt

from IPython.utils import openpy

mydir = os.path.dirname(__file__)
nonascii_path = os.path.join(mydir, "../../core/tests/nonascii.py")


def test_detect_encoding():
    with open(nonascii_path, "rb") as f:
        enc, lines = openpy.detect_encoding(f.readline)
    nt.assert_equal(enc, "iso-8859-5")


def test_read_file():
    with io.open(nonascii_path, encoding="iso-8859-5") as f:
        read_specified_enc = f.read()
    read_detected_enc = openpy.read_py_file(nonascii_path, skip_encoding_cookie=False)
    nt.assert_equal(read_detected_enc, read_specified_enc)
    assert "coding: iso-8859-5" in read_detected_enc

    read_strip_enc_cookie = openpy.read_py_file(
        nonascii_path, skip_encoding_cookie=True
    )
    assert "coding: iso-8859-5" not in read_strip_enc_cookie


def test_source_to_unicode():
    with io.open(nonascii_path, "rb") as f:
        source_bytes = f.read()
    nt.assert_equal(
        openpy.source_to_unicode(source_bytes, skip_encoding_cookie=False).splitlines(),
        source_bytes.decode("iso-8859-5").splitlines(),
    )

    source_no_cookie = openpy.source_to_unicode(source_bytes, skip_encoding_cookie=True)
    nt.assert_not_in("coding: iso-8859-5", source_no_cookie)
{'fraction_non_alphanumeric': 0.11244204018547141, 'fraction_numerical': 0.023183925811437404, 'mean_word_length': 4.39375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3203064', 'n_tokens_mistral': 985, 'n_tokens_neox': 896, 'n_words': 155}