<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import unittest
import vector


class TestTurkish(unittest.TestCase):
    def setUp(self):
        self.vwis = vector.VectorsWithIPASpace('tur-Latn',
                                               'tur-with_attached_suffixes-space')

    def testPunc(self):
        punc_words = [u'"', u"'", u'.', u',', u':', u';', u"‘", u"’", u"”", u"“", u"。", u"，"]
        for word in punc_words:
            print(u'Item:\t{}'.format(word))
            segs = self.vwis.word_to_segs(word)
            self.assertEqual(segs[0][0], u'P')
            self.assertEqual(segs[0][1], 0)
            self.assertIn(segs[0][3], u"\"';,.")

    def testPuncLike(self):
        punc_words = [u"ʼ"]
        for word in punc_words:
            print(u'Item:\t{}'.format(word))
            segs = self.vwis.word_to_segs(word)
            self.assertNotEqual(segs[0][0], u'P')

    def testHaziranda(self):
        word = u'Haziranʼda'
        segs = self.vwis.word_to_segs(word)
        marks = [s[:4] for s in segs]
        correct = [('L', 1, 'H', u'h'),
                   ('L', 0, 'a', u'a'),
                   ('L', 0, 'z', u'z'),
                   ('L', 0, 'i', u'i'),
                   ('L', 0, 'r', u'ɾ'),
                   ('L', 0, 'a', u'a'),
                   ('L', 0, 'n', u'n'),
                   ('P', 0, "'", u''),
                   ('L', 0, 'd', u'd'),
                   ('L', 0, 'a', u'a'), ]
        for m, c in zip(marks, correct):
            self.assertEqual(m, c)

    def testHazirandaPrime(self):
        word = u'Haziranʼda'
        segs = self.vwis.word_to_segs(word)
        marks = [s[:4] for s in segs]
        correct = [('L', 1, 'H', u'h'),
                   ('L', 0, 'a', u'a'),
                   ('L', 0, 'z', u'z'),
                   ('L', 0, 'i', u'i'),
                   ('L', 0, 'r', u'ɾ'),
                   ('L', 0, 'a', u'a'),
                   ('L', 0, 'n', u'n'),
                   ('P', 0, "'", u''),
                   ('L', 0, 'd', u'd'),
                   ('L', 0, 'a', u'a'), ]
        for m, c in zip(marks, correct):
            self.assertEqual(m, c)<commit_msg>Made string literals unicode<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import unittest
import vector


class TestTurkish(unittest.TestCase):
    def setUp(self):
        self.vwis = vector.VectorsWithIPASpace('tur-Latn',
                                               'tur-with_attached_suffixes-space')

    def testPunc(self):
        punc_words = [u'"', u"'", u'.', u',', u':', u';', u"‘", u"’", u"”", u"“", u"。", u"，"]
        for word in punc_words:
            segs = self.vwis.word_to_segs(word)
            self.assertEqual(segs[0][0], u'P')
            self.assertEqual(segs[0][1], 0)
            self.assertIn(segs[0][3], u"\"';,.")

    def testPuncLike(self):
        punc_words = [u"ʼ"]
        for word in punc_words:
            segs = self.vwis.word_to_segs(word)
            self.assertEqual(segs[0][0], u'P')

    def testHaziranda(self):
        word = u'Haziranʼda'
        segs = self.vwis.word_to_segs(word)
        marks = [s[:4] for s in segs]
        correct = [(u'L', 1, u'H', u'h'),
                   (u'L', 0, u'a', u'a'),
                   (u'L', 0, u'z', u'z'),
                   (u'L', 0, u'i', u'i'),
                   (u'L', 0, u'r', u'ɾ'),
                   (u'L', 0, u'a', u'a'),
                   (u'L', 0, u'n', u'n'),
                   (u'P', 0, u"'", u''),
                   (u'L', 0, u'd', u'd'),
                   (u'L', 0, u'a', u'a'), ]
        for m, c in zip(marks, correct):
            self.assertEqual(m, c)

    def testHazirandaPrime(self):
        word = u'Haziranʼda'
        segs = self.vwis.word_to_segs(word)
        marks = [s[:4] for s in segs]
        correct = [(u'L', 1, u'H', u'h'),
                   (u'L', 0, u'a', u'a'),
                   (u'L', 0, u'z', u'z'),
                   (u'L', 0, u'i', u'i'),
                   (u'L', 0, u'r', u'ɾ'),
                   (u'L', 0, u'a', u'a'),
                   (u'L', 0, u'n', u'n'),
                   (u'P', 0, u"'", u''),
                   (u'L', 0, u'd', u'd'),
                   (u'L', 0, u'a', u'a'), ]
        for m, c in zip(marks, correct):
            self.assertEqual(m, c)

    class TestUzbek(unittest.TestCase):
        def setUp(self):
            self.vwis = vector.VectorsWithIPASpace('tur-Latn',
                                                   'tur-with_attached_suffixes-space')

        def test_apostrophe_letter(self):
            pass
{'fraction_non_alphanumeric': 0.20538793103448275, 'fraction_numerical': 0.013793103448275862, 'mean_word_length': 1.484475374732334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1166661', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1687, 'n_words': 372}