<commit_before># -*- coding: utf-8 -*-
import contextlib
import unittest

from korean import *


class TestCase(unittest.TestCase):

    def equal(self, *args, **kwargs):
        return self.assertEqual(*args, **kwargs)

    def same(self, *args, **kwargs):
        return self.assertIs(*args, **kwargs)


class ParticleTestCase(TestCase):

    def test_allomorpheme(self):
        # case clitics
        self.same(Particle(u'가'), Particle(u'이'))
        self.same(Particle(u'를'), Particle(u'을'))
        self.same(Particle(u'로'), Particle(u'으로'))
        self.same(Particle(u'와'), Particle(u'과'))
        self.same(Particle(u'랑'), Particle(u'이랑'))
        # informational clitics
        self.same(Particle(u'는'), Particle(u'은'))
        self.same(Particle(u'나'), Particle(u'이나'))

    def test_inflection(self):
        P, N = Particle, Noun
        self.equal(u'이', inflect(P(u'가'), suffix_of=N(u'받침')))
        self.equal(u'가', inflect(P(u'가'), suffix_of=N(u'나비')))
        self.equal(u'로', inflect(P(u'로'), suffix_of=N(u'마을')))
        self.equal(u'으로', inflect(P(u'로'), suffix_of=N(u'파이썬')))
        self.equal(u'이다', inflect(P(u'다'), suffix_of=N(u'파이썬')))

    def test_naive(self):
        self.assertItemsEqual((u'을(를)', u'를(을)'), Particle(u'을').naive())
        self.equal((u'(으)로',), Particle(u'로').naive())

    def test_proofreading(self):
        self.equal(u'사과는 맛있다.', l10n.proofread(u'사과은(는) 맛있다.'))
        self.equal(u'용사는 검을 획득했다.',
                   l10n.proofread(u'용사은(는) 검을(를) 획득했다.'))


class NounTestCase(TestCase):

    def test_number(self):
        self.equal(u'레벨 사', Noun(u'레벨 4').read())
        self.equal(u'레벨 오십', Noun(u'레벨 50').read())
        self.equal(u'육십사렙', Noun(u'64렙').read())

    def test_unicode_format(self):
        self.equal(u'소년    ', u'{0:6}'.format(Noun(u'소년')))
        self.equal(u'  소녀  ', u'{0:^6}'.format(Noun(u'소녀')))
        self.equal(u'   한국어', u'{0:>6}'.format(Noun(u'한국어')))

    def test_particle_format(self):
        self.equal(u'소년은', u'{0:는}'.format(Noun(u'소년')))
        self.equal(u'소녀는', u'{0:는}'.format(Noun(u'소녀')))
        self.equal(u'한국어를', u'{0:을}'.format(Noun(u'한국어')))
        self.equal(u'레벨 2가', u'{0:이}'.format(Noun(u'레벨 2')))

    def test_undefined_particle_format(self):
        self.equal(u'소년에게', u'{0:에게}'.format(Noun(u'소년')))

    def test_null_format(self):
        self.equal(u'소년', u'{0}'.format(Noun(u'소년')))

    def test_guessable_particle_format(self):<commit_msg>add test<commit_after># -*- coding: utf-8 -*-
import contextlib
import unittest

from korean import *


class TestCase(unittest.TestCase):

    def equal(self, *args, **kwargs):
        return self.assertEqual(*args, **kwargs)

    def same(self, *args, **kwargs):
        return self.assertIs(*args, **kwargs)


class ParticleTestCase(TestCase):

    def test_allomorpheme(self):
        # case clitics
        self.same(Particle(u'가'), Particle(u'이'))
        self.same(Particle(u'를'), Particle(u'을'))
        self.same(Particle(u'로'), Particle(u'으로'))
        self.same(Particle(u'와'), Particle(u'과'))
        self.same(Particle(u'랑'), Particle(u'이랑'))
        # informational clitics
        self.same(Particle(u'는'), Particle(u'은'))
        self.same(Particle(u'나'), Particle(u'이나'))

    def test_inflection(self):
        P, N = Particle, Noun
        self.equal(u'이', inflect(P(u'가'), suffix_of=N(u'받침')))
        self.equal(u'가', inflect(P(u'가'), suffix_of=N(u'나비')))
        self.equal(u'로', inflect(P(u'로'), suffix_of=N(u'마을')))
        self.equal(u'으로', inflect(P(u'로'), suffix_of=N(u'파이썬')))
        self.equal(u'이다', inflect(P(u'다'), suffix_of=N(u'파이썬')))

    def test_naive(self):
        self.assertItemsEqual((u'을(를)', u'를(을)'), Particle(u'을').naive())
        self.equal((u'(으)로',), Particle(u'로').naive())

    def test_proofreading(self):
        self.equal(u'사과는 맛있다.', l10n.proofread(u'사과은(는) 맛있다.'))
        self.equal(u'집으로 가자.', l10n.proofread(u'집(으)로 가자.'))
        self.equal(u'용사는 검을 획득했다.',
                   l10n.proofread(u'용사은(는) 검을(를) 획득했다.'))


class NounTestCase(TestCase):

    def test_number(self):
        self.equal(u'레벨 사', Noun(u'레벨 4').read())
        self.equal(u'레벨 오십', Noun(u'레벨 50').read())
        self.equal(u'육십사렙', Noun(u'64렙').read())

    def test_unicode_format(self):
        self.equal(u'소년    ', u'{0:6}'.format(Noun(u'소년')))
        self.equal(u'  소녀  ', u'{0:^6}'.format(Noun(u'소녀')))
        self.equal(u'   한국어', u'{0:>6}'.format(Noun(u'한국어')))

    def test_particle_format(self):
        self.equal(u'소년은', u'{0:는}'.format(Noun(u'소년')))
        self.equal(u'소녀는', u'{0:는}'.format(Noun(u'소녀')))
        self.equal(u'한국어를', u'{0:을}'.format(Noun(u'한국어')))
        self.equal(u'레벨 2가', u'{0:이}'.format(Noun(u'레벨 2')))

    def test_undefined_particle_format(self):
        self.equal(u'소년에게', u'{0:에게}'.format(Noun(u'소년')))

    def test_null_format(self):
        self.equal(u'소년', u'{0}'.format(Noun(u'소년')))

    def test_guessable_particle_format(self):
{'fraction_non_alphanumeric': 0.21818546768366118, 'fraction_numerical': 0.010036130068245684, 'mean_word_length': 3.4253996447602133, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '544894', 'n_tokens_mistral': 2265, 'n_tokens_neox': 2401, 'n_words': 272}