<commit_before>__author__ = 'allentran'

import numpy as np

from ..scraper import Scraper
from ..model import lstm

def scraper_test():

    scraper = Scraper()
    assert len(scraper.get_docs(limit=1)) == 1

def model_test():

    n_batch = 10
    T = 20

    test_ob = dict(
        word_vectors=np.random.standard_normal((T, n_batch, 300)),
        rates=np.ones((n_batch, 3)),
        max_mask=np.ones((T, n_batch)),
        regimes=np.ones(n_batch),
        doc_types=np.ones(n_batch)
    )

    model = lstm.FedLSTM(
        hidden_sizes=[10, 10, 10, 10]
    )

    model.get_cost_and_update(test_ob)<commit_msg>test bug<commit_after>__author__ = 'allentran'

import numpy as np

from ..scraper import Scraper
from ..model import lstm

def scraper_test():

    scraper = Scraper()
    assert len(scraper.get_docs(limit=1)) == 1

def model_test():

    n_batch = 10
    T = 20

    test_ob = dict(
        word_vectors=np.random.standard_normal((T, n_batch, 300)),
        rates=np.ones((n_batch, 3)),
        max_mask=np.ones((T, n_batch)),
        regimes=np.ones(n_batch),
        doc_types=np.ones(n_batch)
    )

    model = lstm.FedLSTM(
        hidden_sizes=[10, 10, 10, 10]
    )

    model.get_cost_and_update(
        test_ob['word_vectors'],
        test_ob['rates'],
        test_ob['max_mask'],
        test_ob['regimes'],
        test_ob['doc_types']
    )
{'fraction_non_alphanumeric': 0.13372093023255813, 'fraction_numerical': 0.02616279069767442, 'mean_word_length': 2.7216216216216216, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6474030', 'n_tokens_mistral': 562, 'n_tokens_neox': 514, 'n_words': 95}