<commit_before># -*- coding: utf-8 -*-
import pytest
from smartformat import SmartFormatter
from smartformat.ext.korean import ko
from smartformat.ext.korean.hangul import join_phonemes, split_phonemes


@pytest.fixture
def smart():
    formatter = SmartFormatter()
    formatter.register([ko])
    return formatter


def test_split_phonemes():
    assert split_phonemes(u'쏚') == (u'ㅆ', u'ㅗ', u'ㄲ')
    assert split_phonemes(u'섭') == (u'ㅅ', u'ㅓ', u'ㅂ')
    assert split_phonemes(u'투') == (u'ㅌ', u'ㅜ', u'')
    assert split_phonemes(u'투', onset=False) == (None, u'ㅜ', u'')
    with pytest.raises(ValueError):
        split_phonemes(u'X')
    with pytest.raises(ValueError):
        split_phonemes(u'섭섭')


def test_join_phonemes():
    assert join_phonemes(u'ㅅ', u'ㅓ', u'ㅂ') == u'섭'
    assert join_phonemes((u'ㅅ', u'ㅓ', u'ㅂ')) == u'섭'
    assert join_phonemes(u'ㅊ', u'ㅠ') == u'츄'
    assert join_phonemes(u'ㅊ', u'ㅠ', u'') == u'츄'
    assert join_phonemes((u'ㅊ', u'ㅠ')) == u'츄'
    with pytest.raises(TypeError):
        join_phonemes(u'ㄷ', u'ㅏ', u'ㄹ', u'ㄱ')


def test_explicit(smart):
    f = smart.format
    assert f(u'{:ko(아):{}} 안녕', u'피카츄') == u'피카츄야 안녕'
    assert f(u'{:ko(아):{}} 안녕', u'버터플') == u'버터플아 안녕'
    assert f(u'{:ko(아):{}} 안녕', u'고라파덕') == u'고라파덕아 안녕'
    assert f(u'{:ko(을):*{}*} 깎는다.', u'사과') == u'*사과*를 깎는다.'
    assert f(u'{:ko(을):}', u'수박') == u'을'


def test_implicit(smart):
    f = smart.format
    assert f(u'{:아} 안녕', u'피카츄') == u'피카츄야 안녕'
    assert f(u'{:아} 안녕', u'버터플') == u'버터플아 안녕'
    assert f(u'{:아} 안녕', u'고라파덕') == u'고라파덕아 안녕'
    assert f(u'{:을} 칼로 깎는다.', u'사과') == u'사과를 칼로 깎는다.'
    assert f(u'{:-을}', u'수박') == u'을'
    assert f(u'{:ㄱ나니?}', u'서태지') == u'서태지'
    # All option letters have to be Hangul.
    assert f(u'{:을!}', u'서태지') == u'서태지'


def test_euro(smart):
    f = smart.format
    assert f(u'{:ko(으로):{}}', u'피카츄') == u'피카츄로'
    assert f(u'{:ko(으로):{}}', u'버터플') == u'버터플로'
    assert f(u'{:ko(으로):{}}', u'고라파덕') == u'고라파덕으로'
    assert f(u'{:ko(으로):{}}', u'Pikachu') == u'Pikachu(으)로'
    assert f(u'{:ko(로서):{}}', u'피카츄') == u'피카츄로서'
    assert f(u'{:ko(로서):{}}', u'버터플') == u'버터플로서'
    assert f(u'{:ko(로서):{}}', u'고라파덕') == u'고라파덕으로서'
    assert f(u'{:ko(로써):{}}', u'피카츄') == u'피카츄로써'
    assert f(u'{:ko(로써):{}}', u'버터플') == u'버터플로써'
    assert f(u'{:ko(로써):{}}', u'고라파덕') == u'고라파덕으로써'
    assert f(u'{:ko(로부터):{}}', u'피카츄') == u'피카츄로부터'
    assert f(u'{:ko(로부터):{}}', u'버터플') == u'버터플로부터'
    assert f(u'{:ko(로부터):{}}', u'고라파덕') == u'고라파덕으로부터'
    assert f(u'{:ko(로부터도):{}}', u'고라파덕') == u'고라파덕으로부터도'
    assert f(u'{:ko((으)로부터의):{}} 편지', u'그녀') == u'그녀로부터의 편지'


def test_exceptions(smart):
    f = smart.format
    # Empty.
    assert f(u'{:를}', u'') == u'을(를)'
    # Onsets only.
    assert f(u'{:를}', u'ㅋㅋㅋ') == u'ㅋㅋㅋ을(를)'


def test_blind(smart):
    f = smart.format
    assert f(u'{:ko(으로):{}}', u'피카츄(Lv.25)') == u'피카츄(Lv.25)로'
    assert f(u'{:ko(으로):{}}', u'피카(?)츄') == u'피카(?)츄로'
    assert f(u'{:ko(으로):{}}', u'헬로월드!') == u'헬로월드!로'
    assert f(u'{:ko(으로):{}}', u'?_?') == u'?_?(으)로'
    assert f(u'{:가}?', u'임창정,,,') == u'임창정,,,이?'


def test_vocative_particles(smart):
    f = smart.format
    assert f(u'{:야}', u'친구') == u'친구야'
    assert f(u'{:야}', u'사랑') == u'사랑아'
    assert f(u'{:아}', u'사랑') == u'사랑아'
    assert f(u'{:여}', u'친구') == u'친구여'
    assert f(u'{:여}', u'사랑') == u'사랑이여'
    assert f(u'{:이시여}', u'하늘') == u'하늘이시여'
    assert f(u'{:이시여}', u'바다') == u'바다시여'


def test_ida(smart):
    """Cases for '이다' which is a copulative and existential verb."""
    f = smart.format
    # Do or don't inject '이'.
    assert f(u'{:이다}', u'피카츄') == u'피카츄다'
    assert f(u'{:이다}', u'버터플') == u'버터플이다'
    # Merge with the following vowel as /j/.
    assert f(u'{:이에요}', u'피카츄') == u'피카츄예요'
    assert f(u'{:이에요}', u'버터플') == u'버터플이에요'
    # No allomorphs.
    assert f(u'{:입니다}', u'피카츄') == u'피카츄입니다'<commit_msg>Add a test case<commit_after># -*- coding: utf-8 -*-
import pytest
from smartformat import SmartFormatter
from smartformat.ext.korean import ko
from smartformat.ext.korean.hangul import join_phonemes, split_phonemes


@pytest.fixture
def smart():
    formatter = SmartFormatter()
    formatter.register([ko])
    return formatter


def test_split_phonemes():
    assert split_phonemes(u'쏚') == (u'ㅆ', u'ㅗ', u'ㄲ')
    assert split_phonemes(u'섭') == (u'ㅅ', u'ㅓ', u'ㅂ')
    assert split_phonemes(u'투') == (u'ㅌ', u'ㅜ', u'')
    assert split_phonemes(u'투', onset=False) == (None, u'ㅜ', u'')
    with pytest.raises(ValueError):
        split_phonemes(u'X')
    with pytest.raises(ValueError):
        split_phonemes(u'섭섭')


def test_join_phonemes():
    assert join_phonemes(u'ㅅ', u'ㅓ', u'ㅂ') == u'섭'
    assert join_phonemes((u'ㅅ', u'ㅓ', u'ㅂ')) == u'섭'
    assert join_phonemes(u'ㅊ', u'ㅠ') == u'츄'
    assert join_phonemes(u'ㅊ', u'ㅠ', u'') == u'츄'
    assert join_phonemes((u'ㅊ', u'ㅠ')) == u'츄'
    with pytest.raises(TypeError):
        join_phonemes(u'ㄷ', u'ㅏ', u'ㄹ', u'ㄱ')


def test_explicit(smart):
    f = smart.format
    assert f(u'{:ko(아):{}} 안녕', u'피카츄') == u'피카츄야 안녕'
    assert f(u'{:ko(아):{}} 안녕', u'버터플') == u'버터플아 안녕'
    assert f(u'{:ko(아):{}} 안녕', u'고라파덕') == u'고라파덕아 안녕'
    assert f(u'{:ko(을):*{}*} 깎는다.', u'사과') == u'*사과*를 깎는다.'
    assert f(u'{:ko(을):}', u'수박') == u'을'


def test_implicit(smart):
    f = smart.format
    assert f(u'{:아} 안녕', u'피카츄') == u'피카츄야 안녕'
    assert f(u'{:아} 안녕', u'버터플') == u'버터플아 안녕'
    assert f(u'{:아} 안녕', u'고라파덕') == u'고라파덕아 안녕'
    assert f(u'{:을} 칼로 깎는다.', u'사과') == u'사과를 칼로 깎는다.'
    assert f(u'{:-을}', u'수박') == u'을'
    assert f(u'{:ㄱ나니?}', u'서태지') == u'서태지'
    # All option letters have to be Hangul.
    assert f(u'{:을!}', u'서태지') == u'서태지'


def test_euro(smart):
    f = smart.format
    assert f(u'{:ko(으로):{}}', u'피카츄') == u'피카츄로'
    assert f(u'{:ko(으로):{}}', u'버터플') == u'버터플로'
    assert f(u'{:ko(으로):{}}', u'고라파덕') == u'고라파덕으로'
    assert f(u'{:ko(으로):{}}', u'Pikachu') == u'Pikachu(으)로'
    assert f(u'{:ko(로서):{}}', u'피카츄') == u'피카츄로서'
    assert f(u'{:ko(로서):{}}', u'버터플') == u'버터플로서'
    assert f(u'{:ko(로서):{}}', u'고라파덕') == u'고라파덕으로서'
    assert f(u'{:ko(로써):{}}', u'피카츄') == u'피카츄로써'
    assert f(u'{:ko(로써):{}}', u'버터플') == u'버터플로써'
    assert f(u'{:ko(로써):{}}', u'고라파덕') == u'고라파덕으로써'
    assert f(u'{:ko(로부터):{}}', u'피카츄') == u'피카츄로부터'
    assert f(u'{:ko(로부터):{}}', u'버터플') == u'버터플로부터'
    assert f(u'{:ko(로부터):{}}', u'고라파덕') == u'고라파덕으로부터'
    assert f(u'{:ko(로부터도):{}}', u'고라파덕') == u'고라파덕으로부터도'
    assert f(u'{:ko((으)로부터의):{}} 편지', u'그녀') == u'그녀로부터의 편지'


def test_exceptions(smart):
    f = smart.format
    # Empty.
    assert f(u'{:를}', u'') == u'을(를)'
    # Onsets only.
    assert f(u'{:를}', u'ㅋㅋㅋ') == u'ㅋㅋㅋ을(를)'


def test_blind(smart):
    f = smart.format
    assert f(u'{:ko(으로):{}}', u'피카츄(Lv.25)') == u'피카츄(Lv.25)로'
    assert f(u'{:ko(으로):{}}', u'피카(?)츄') == u'피카(?)츄로'
    assert f(u'{:ko(으로):{}}', u'헬로월드!') == u'헬로월드!로'
    assert f(u'{:ko(으로):{}}', u'?_?') == u'?_?(으)로'
    assert f(u'{:가}?', u'임창정,,,') == u'임창정,,,이?'
    assert f(u'{:을} 샀다.', u'<듀랑고>') == u'<듀랑고>를 샀다.'


def test_vocative_particles(smart):
    f = smart.format
    assert f(u'{:야}', u'친구') == u'친구야'
    assert f(u'{:야}', u'사랑') == u'사랑아'
    assert f(u'{:아}', u'사랑') == u'사랑아'
    assert f(u'{:여}', u'친구') == u'친구여'
    assert f(u'{:여}', u'사랑') == u'사랑이여'
    assert f(u'{:이시여}', u'하늘') == u'하늘이시여'
    assert f(u'{:이시여}', u'바다') == u'바다시여'


def test_ida(smart):
    """Cases for '이다' which is a copulative and existential verb."""
    f = smart.format
    # Do or don't inject '이'.
    assert f(u'{:이다}', u'피카츄') == u'피카츄다'
    assert f(u'{:이다}', u'버터플') == u'버터플이다'
    # Merge with the following vowel as /j/.
    assert f(u'{:이에요}', u'피카츄') == u'피카츄예요'
    assert f(u'{:이에요}', u'버터플') == u'버터플이에요'
    # No allomorphs.
    assert f(u'{:입니다}', u'피카츄') == u'피카츄입니다'
{'fraction_non_alphanumeric': 0.2764788552446659, 'fraction_numerical': 0.0012776287210936501, 'mean_word_length': 3.817230769230769, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '26799', 'n_tokens_mistral': 4688, 'n_tokens_neox': 5427, 'n_words': 712}