<commit_before># -*- coding: utf-8 -*-

import unittest
import _panphon


class TestIpaRe(unittest.TestCase):

    def setUp(self):
        self.ft = _panphon.FeatureTable()

    def test_compile_regex_from_str1(self):
        r = self.ft.compile_regex_from_str('[-son -cont][+syl -hi -lo]')
        self.assertIsNotNone(r.match(u'tʰe'))
        self.assertIsNone(r.match(u'pi'))

    def test_compile_regex_from_str2(self):
        r = self.ft.compile_regex_from_str(u'[-son -cont][+son +cont]')
        self.assertIsNotNone(r.match(u'pj'))
        self.assertIsNone(r.match(u'ts'))

class TestSonority(unittest.TestCase):

    def setUp(self):
        self.ft = _panphon.FeatureTable()

    def test_sonority_one_segment(self):
        self.assertEqual(self.ft.sonority(u'a'), 9)

    def test_sonority_basic(self):
        segs = [u'a', u'e', u'j', u'l', u'n', u'ʒ', u's', u'b', u't', ]
        scores = [9, 8, 7, 6, 5, 4, 3, 2, 1]
        self.assertEqual(map(self.ft.sonority, segs), scores)

if __name__ == '__main__':<commit_msg>Updated tests for new sonority scale<commit_after># -*- coding: utf-8 -*-

import unittest
import _panphon


class TestIpaRe(unittest.TestCase):

    def setUp(self):
        self.ft = _panphon.FeatureTable()

    def test_compile_regex_from_str1(self):
        r = self.ft.compile_regex_from_str('[-son -cont][+syl -hi -lo]')
        self.assertIsNotNone(r.match(u'tʰe'))
        self.assertIsNone(r.match(u'pi'))

    def test_compile_regex_from_str2(self):
        r = self.ft.compile_regex_from_str(u'[-son -cont][+son +cont]')
        self.assertIsNotNone(r.match(u'pj'))
        self.assertIsNone(r.match(u'ts'))


class TestSonority(unittest.TestCase):

    def setUp(self):
        self.ft = _panphon.FeatureTable()

    def test_sonority_nine(self):
        segs = [u'a', u'ɑ', u'æ', u'ɒ', u'e', u'o']
        scores = [9] * 6
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_eight(self):
        segs = [u'i', u'u', u'ɨ', u'ʉ', u'ɯ', u'u']
        scores = [8] * 6
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_seven(self):
        segs = [u'j', u'w', u'ʋ', u'ɰ', u'ɹ']
        scores = [7] * 5
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_six(self):
        segs = [u'l', u'ɭ', u'r', u'ɾ']
        scores = [6] * 4
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_five(self):
        segs = [u'n', u'm', u'ŋ', u'ɴ']
        scores = [5] * 4
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_four(self):
        segs = [u'v', u'z', u'ʒ', u'ɣ']
        scores = [4] * 4
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_three(self):
        segs = [u'f', u's', u'x', u'ħ', u'ʃ']
        scores = [3] * 5
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_two(self):
        segs = [u'b', u'g', u'd', u'ɢ']
        scores = [2] * 4
        self.assertEqual(map(self.ft.sonority, segs), scores)

    def test_sonority_one(self):
        segs = [u'p', u'k', u'c', u'q']
        scores = [1] * 4
        self.assertEqual(map(self.ft.sonority, segs), scores)

if __name__ == '__main__':
{'fraction_non_alphanumeric': 0.17494666260286498, 'fraction_numerical': 0.010362694300518135, 'mean_word_length': 2.856639247943596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3272006', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1271, 'n_words': 234}