<commit_before>"""Tests for embeddings module."""

import unittest
from vsmlib.corpus import load_file_as_ids, FileTokenIterator, DirTokenIterator
from vsmlib.vocabulary import Vocabulary

# todo: use local vocab
path_vocab = "./test/data/vocab"
path_text = "./test/data/corpora/plain"
path_gzipped = "./test/data/corpora/gzipped"
path_text_file = "./test/data/corpora/plain/sense_small.txt"


class Tests(unittest.TestCase):

    def test_file_iter(self):
        cnt = 0
        for w in (FileTokenIterator(path_text_file)):
            cnt += 1
        print(cnt, "words read")

    def test_dir_iter(self):
        cnt = 0
        for w in (DirTokenIterator(path_text)):
            cnt += 1
        print(cnt, "words read")

    def test_text_to_ids(self):
        v = Vocabulary()
        v.load(path_vocab)
        doc = load_file_as_ids(path_text_file, v)
        print(doc.shape)
        print(doc[:10])

    def test_dir_iter_gzipped(self):
        cnt = 0
        for w in (DirTokenIterator(path_gzipped)):
            cnt += 1
        print(cnt, "words read")<commit_msg>corpus test print sample of parsed tokens<commit_after>"""Tests for embeddings module."""

import unittest
from vsmlib.corpus import load_file_as_ids, FileTokenIterator, DirTokenIterator
from vsmlib.vocabulary import Vocabulary

# todo: use local vocab
path_vocab = "./test/data/vocab"
path_text = "./test/data/corpora/plain"
path_gzipped = "./test/data/corpora/gzipped"
path_text_file = "./test/data/corpora/plain/sense_small.txt"


class Tests(unittest.TestCase):

    def test_file_iter(self):
        cnt = 0
        for w in (FileTokenIterator(path_text_file)):
            if cnt < 10:
                print(w)
            cnt += 1
        print(cnt, "words read")

    def test_dir_iter(self):
        cnt = 0
        for w in (DirTokenIterator(path_text)):
            cnt += 1
        print(cnt, "words read")

    def test_text_to_ids(self):
        v = Vocabulary()
        v.load(path_vocab)
        doc = load_file_as_ids(path_text_file, v)
        print(doc.shape)
        print(doc[:10])

    def test_dir_iter_gzipped(self):
        cnt = 0
        for w in (DirTokenIterator(path_gzipped)):
            cnt += 1
        print(cnt, "words read")
{'fraction_non_alphanumeric': 0.11051693404634581, 'fraction_numerical': 0.008021390374331552, 'mean_word_length': 2.620967741935484, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3667665', 'n_tokens_mistral': 770, 'n_tokens_neox': 733, 'n_words': 167}