<commit_before># -*- coding: utf-8 -*-

import pytest

import os
import sys

import numpy as np

from inferbeddings.io import iopen, load_glove, load_word2vec


def test_load_glove_large():
    glove_path = os.path.expanduser('~/data/glove/glove.840B.300d.txt')
    if os.path.isfile(glove_path):
        word_set = {'machine', 'learning'}
        with open(glove_path, 'rb') as stream:
            word_to_embedding = load_glove(stream=stream, words=word_set)

        machine_vector = word_to_embedding['machine']
        learning_vector = word_to_embedding['learning']

        np.testing.assert_allclose(machine_vector[0:3], [-0.12538, 0.38888, 0.48011], rtol=1e-3)
        np.testing.assert_allclose(learning_vector[0:3], [0.047511, 0.1404, -0.11736], rtol=1e-3)


def test_load_word2vec():
    word2vec_path = os.path.expanduser('~/data/word2vec/GoogleNews-vectors-negative300.bin.gz')
    if os.path.isfile(word2vec_path):
        word_set = {'machine', 'learning'}
        with iopen(word2vec_path, 'rb') as stream:
            word_to_embedding = load_word2vec(stream=stream, words=word_set)

        print(word_to_embedding)

        machine_vector = word_to_embedding['machine']
        learning_vector = word_to_embedding['learning']

        print(machine_vector)
        print(learning_vector)


def test_load_glove():
    glove_path = 'data/glove/glove.6B.50d.txt.gz'
    if os.path.isfile(glove_path):
        word_set = {'house'}
        with iopen(glove_path, 'r') as stream:
            model = load_glove(stream=stream, words=word_set)
        assert 'house' in model
        assert 0.60136 < model['house'][0] < 0.60138


if __name__ == '__main__':
    pytest.main([__file__])
<commit_msg>pre-trained embeddings - loading and testing<commit_after># -*- coding: utf-8 -*-

import pytest

import os
import sys

import numpy as np

from inferbeddings.io import iopen, load_glove, load_word2vec


def test_load_glove_large():
    glove_path = os.path.expanduser('~/data/glove/glove.840B.300d.txt')
    if os.path.isfile(glove_path):
        word_set = {'machine', 'learning'}
        with open(glove_path, 'r') as stream:
            word_to_embedding = load_glove(stream=stream, words=word_set)

        machine_vector = word_to_embedding['machine']
        learning_vector = word_to_embedding['learning']

        np.testing.assert_allclose(machine_vector[0:3], [-0.12538, 0.38888, 0.48011], rtol=1e-3)
        np.testing.assert_allclose(learning_vector[0:3], [0.047511, 0.1404, -0.11736], rtol=1e-3)


def test_load_word2vec():
    word2vec_path = os.path.expanduser('~/data/word2vec/GoogleNews-vectors-negative300.bin.gz')
    if os.path.isfile(word2vec_path):
        word_set = {'machine', 'learning'}
        with iopen(word2vec_path, 'rb') as stream:
            word_to_embedding = load_word2vec(stream=stream, words=word_set)

        print(word_to_embedding)

        machine_vector = word_to_embedding['machine']
        learning_vector = word_to_embedding['learning']

        print(machine_vector)
        print(learning_vector)


def test_load_glove():
    glove_path = 'data/glove/glove.6B.50d.txt.gz'
    if os.path.isfile(glove_path):
        word_set = {'house'}
        with iopen(glove_path, 'r') as stream:
            model = load_glove(stream=stream, words=word_set)
        assert 'house' in model
        assert 0.60136 < model['house'][0] < 0.60138


if __name__ == '__main__':
    pytest.main([__file__])

{'fraction_non_alphanumeric': 0.12811594202898552, 'fraction_numerical': 0.04463768115942029, 'mean_word_length': 3.39058524173028, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4103084', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1221, 'n_words': 208}