<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from ftfy import fix_text
from ftfy.fixes import fix_encoding_and_explain, apply_plan
from nose.tools import eq_


TEST_CASES = [
    ## These are excerpts from text actually seen in the wild, mostly on
    ## Twitter. Usernames and links have been removed.
    ("He's Justinâ¤", "He's Justin❤"),
    ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.",
     "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."),
    ("âœ” No problems", "✔ No problems"),
    ('4288×…', '4288×…'),
    ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ…'),
    ('PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…', 'PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…'),
    ('TEM QUE SEGUIR, SDV SÓ…', 'TEM QUE SEGUIR, SDV SÓ…'),
    ('Join ZZAJÉ’s Official Fan List and receive news, events, and more!', "Join ZZAJÉ's Official Fan List and receive news, events, and more!"),
    ('L’épisode 8 est trop fou ouahh', "L'épisode 8 est trop fou ouahh"),
    ("РґРѕСЂРѕРіРµ РР·-РїРѕРґ #С„СѓС‚Р±РѕР»",
     "дороге Из-под #футбол"),
    ("\x84Handwerk bringt dich \xfcberall hin\x93: Von der YOU bis nach Monaco",
     '"Handwerk bringt dich überall hin": Von der YOU bis nach Monaco'),
    ("Hi guys í ½í¸", "Hi guys 😍"),
    ("hihi RT username: âºí ½í¸",
     "hihi RT username: ☺😘"),
    ("Beta Haber: HÄ±rsÄ±zÄ± BÃ¼yÃ¼ Korkuttu",
     "Beta Haber: Hırsızı Büyü Korkuttu"),
    ("Ôôô VIDA MINHA", "Ôôô VIDA MINHA"),
    ('[x]\xa0©', '[x]\xa0©'),
    ('2012—∞', '2012—∞'),
    ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.',
     'Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.'),
    ('Р С—РЎР‚Р С‘РЎРЏРЎвЂљР Р…Р С•РЎРѓРЎвЂљР С‘. РІСњВ¤', 'приятности. ❤'),
    ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New Ã¢â‚¬Å“ RomanceÃ¢â‚¬Â.',
     'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'),
    ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí´´"),
    ('HUHLL Õ…', 'HUHLL Õ…'),
    ('Iggy Pop (nÃƒÂ© <NAME>)', 'Iggy Pop (né <NAME>)'),
    ('eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías',
     'eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías'),
    ("Direzione Pd, ok âsenza modifiche all'Italicum.",
     "Direzione Pd, ok \"senza modifiche\" all'Italicum."),
    ('Engkau masih yg terindah, indah di dalam hatikuâ™«~',
     'Engkau masih yg terindah, indah di dalam hatiku♫~'),
    ('SENSЕ - Oleg Tsedryk', 'SENSЕ - Oleg Tsedryk'),   # this Е is a Ukrainian letter
    ('OK??:(   `¬´    ):', 'OK??:(   `¬´    ):'),
    ("selamat berpuasa sob (Ã\xa0Â¸â€¡'ÃŒâ‚¬Ã¢Å’Â£'ÃŒÂ\x81)Ã\xa0Â¸â€¡",
     "selamat berpuasa sob (ง'̀⌣'́)ง"),
    ("The Mona Lisa doesnÃƒÂ¢Ã¢â€šÂ¬Ã¢â€žÂ¢t have eyebrows.",
     "The Mona Lisa doesn't have eyebrows."),
    ("#╨┐╤Ç╨░╨▓╨╕╨╗╤î╨╜╨╛╨╡╨┐╨╕╤é╨░╨╜╨╕╨╡", "#правильноепитание"),
    ('∆°', '∆°'),

    # Test Windows-1250 mixups
    ("LiĂ¨ge Avenue de l'HĂ´pital", "Liège Avenue de l'Hôpital"),
    ("It was namedÂ â€žscarsÂ´ stonesâ€ś after the rock-climbers who got hurt while climbing on it.",
     "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."),

    # The second test is different in iso-8859-2
    ("It was namedÂ\xa0â\x80\x9escarsÂ´ stonesâ\x80\x9c after the rock-climbers who got hurt while climbing on it.",
     "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."),

    # This one has two differently-broken layers of Windows-1252 <=> UTF-8,
    # and it's kind of amazing that we solve it.
    ('Arsenal v Wolfsburg: pre-season friendly â\x80â\x80\x9c live!',
     'Arsenal v Wolfsburg: pre-season friendly – live!'),

    # Test that we can mostly decode this face when the nonprintable
    # character \x9d is lost
    ('Ã¢â€\x9dâ€™(Ã¢Å’Â£Ã‹â€ºÃ¢Å’Â£)Ã¢â€\x9dÅ½', '┒(⌣˛⌣)┎'),
    ('Ã¢â€�â€™(Ã¢Å’Â£Ã‹â€ºÃ¢Å’Â£)Ã¢â€�Å½', '�(⌣˛⌣)�'),

    # You tried
    ('I just figured out how to tweet emojis! â\x9a½í\xa0½í¸\x80í\xa0½í¸\x81í\xa0½í¸\x82í\xa0½í¸\x86í\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8e',
     'I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎'),

    # Former false positives
    ("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿"),
    ("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"),<commit_msg>found some more test cases for windows-1250<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from ftfy import fix_text
from ftfy.fixes import fix_encoding_and_explain, apply_plan
from nose.tools import eq_


TEST_CASES = [
    ## These are excerpts from text actually seen in the wild, mostly on
    ## Twitter. Usernames and links have been removed.
    ("He's Justinâ¤", "He's Justin❤"),
    ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.",
     "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."),
    ("âœ” No problems", "✔ No problems"),
    ('4288×…', '4288×…'),
    ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ…'),
    ('PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…', 'PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…'),
    ('TEM QUE SEGUIR, SDV SÓ…', 'TEM QUE SEGUIR, SDV SÓ…'),
    ('Join ZZAJÉ’s Official Fan List and receive news, events, and more!', "Join ZZAJÉ's Official Fan List and receive news, events, and more!"),
    ('L’épisode 8 est trop fou ouahh', "L'épisode 8 est trop fou ouahh"),
    ("РґРѕСЂРѕРіРµ РР·-РїРѕРґ #С„СѓС‚Р±РѕР»",
     "дороге Из-под #футбол"),
    ("\x84Handwerk bringt dich \xfcberall hin\x93: Von der YOU bis nach Monaco",
     '"Handwerk bringt dich überall hin": Von der YOU bis nach Monaco'),
    ("Hi guys í ½í¸", "Hi guys 😍"),
    ("hihi RT username: âºí ½í¸",
     "hihi RT username: ☺😘"),
    ("Beta Haber: HÄ±rsÄ±zÄ± BÃ¼yÃ¼ Korkuttu",
     "Beta Haber: Hırsızı Büyü Korkuttu"),
    ("Ôôô VIDA MINHA", "Ôôô VIDA MINHA"),
    ('[x]\xa0©', '[x]\xa0©'),
    ('2012—∞', '2012—∞'),
    ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.',
     'Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.'),
    ('Р С—РЎР‚Р С‘РЎРЏРЎвЂљР Р…Р С•РЎРѓРЎвЂљР С‘. РІСњВ¤', 'приятности. ❤'),
    ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New Ã¢â‚¬Å“ RomanceÃ¢â‚¬Â.',
     'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'),
    ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí´´"),
    ('HUHLL Õ…', 'HUHLL Õ…'),
    ('Iggy Pop (nÃƒÂ© <NAME>)', 'Iggy Pop (né <NAME>)'),
    ('eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías',
     'eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías'),
    ("Direzione Pd, ok âsenza modifiche all'Italicum.",
     "Direzione Pd, ok \"senza modifiche\" all'Italicum."),
    ('Engkau masih yg terindah, indah di dalam hatikuâ™«~',
     'Engkau masih yg terindah, indah di dalam hatiku♫~'),
    ('SENSЕ - <NAME>', 'SENSЕ - <NAME>'),   # this Е is a Ukrainian letter
    ('OK??:(   `¬´    ):', 'OK??:(   `¬´    ):'),
    ("selamat berpuasa sob (Ã\xa0Â¸â€¡'ÃŒâ‚¬Ã¢Å’Â£'ÃŒÂ\x81)Ã\xa0Â¸â€¡",
     "selamat berpuasa sob (ง'̀⌣'́)ง"),
    ("The Mona Lisa doesnÃƒÂ¢Ã¢â€šÂ¬Ã¢â€žÂ¢t have eyebrows.",
     "The Mona Lisa doesn't have eyebrows."),
    ("#╨┐╤Ç╨░╨▓╨╕╨╗╤î╨╜╨╛╨╡╨┐╨╕╤é╨░╨╜╨╕╨╡", "#правильноепитание"),
    ('∆°', '∆°'),

    # Test Windows-1250 mixups
    ("LiĂ¨ge Avenue de l'HĂ´pital", "Liège Avenue de l'Hôpital"),
    ("It was namedÂ â€žscarsÂ´ stonesâ€ś after the rock-climbers who got hurt while climbing on it.",
     "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."),
    ("vedere Ă®nceĹŁoĹźatÄ\x83", "vedere înceţoşată"),
    ("NapĂ\xadĹˇte nĂˇm !", "Napíšte nám !"),

    # The second test is different in iso-8859-2
    ("It was namedÂ\xa0â\x80\x9escarsÂ´ stonesâ\x80\x9c after the rock-climbers who got hurt while climbing on it.",
     "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."),

    # This one has two differently-broken layers of Windows-1252 <=> UTF-8,
    # and it's kind of amazing that we solve it.
    ('Arsenal v Wolfsburg: pre-season friendly â\x80â\x80\x9c live!',
     'Arsenal v Wolfsburg: pre-season friendly – live!'),

    # Test that we can mostly decode this face when the nonprintable
    # character \x9d is lost
    ('Ã¢â€\x9dâ€™(Ã¢Å’Â£Ã‹â€ºÃ¢Å’Â£)Ã¢â€\x9dÅ½', '┒(⌣˛⌣)┎'),
    ('Ã¢â€�â€™(Ã¢Å’Â£Ã‹â€ºÃ¢Å’Â£)Ã¢â€�Å½', '�(⌣˛⌣)�'),

    # You tried
    ('I just figured out how to tweet emojis! â\x9a½í\xa0½í¸\x80í\xa0½í¸\x81í\xa0½í¸\x82í\xa0½í¸\x86í\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8e',
     'I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎'),

    # Former false positives
    ("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿"),
    ("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"),
{'fraction_non_alphanumeric': 0.16704143912624478, 'fraction_numerical': 0.018417389442124423, 'mean_word_length': 3.623762376237624, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '627098', 'n_tokens_mistral': 4318, 'n_tokens_neox': 4250, 'n_words': 1161}