<commit_before>'''Not sure if this is useful -- try training the Tensorizer component.'''
import plac
import spacy
import thinc.extra.datasets
from spacy.util import minibatch
import tqdm


def load_imdb():
    nlp = spacy.blank('en')
    train, dev = thinc.extra.datasets.imdb()
    train_texts, _ = zip(*train)
    dev_texts, _ = zip(*dev)
    nlp.add_pipe(nlp.create_pipe('sentencizer'))
    return list(get_sentences(nlp, train_texts)), list(get_sentences(nlp, dev_texts))


def get_sentences(nlp, texts):
    for doc in nlp.pipe(texts):
        for sent in doc.sents:
            yield sent.text


def main():
    print("Load data")
    train_texts, dev_texts = load_imdb()
    train_texts = train_texts[:1000]
    print("Load vectors")
    nlp = spacy.load('en_vectors_web_lg')
    print("Start training")
    nlp.add_pipe(nlp.create_pipe('tagger'))
    tensorizer = nlp.create_pipe('tensorizer')
    nlp.add_pipe(tensorizer)
    optimizer = nlp.begin_training()

    for i in range(10):
        losses = {}
        for i, batch in enumerate(minibatch(tqdm.tqdm(train_texts))):
            docs = [nlp.make_doc(text) for text in batch]
            tensorizer.update(docs, None, losses=losses, sgd=optimizer, drop=0.5)
            if i % 10 == 0:
                print(losses)

if __name__ == '__main__':
    plac.call(main)
<commit_msg>Improve train tensorizer script<commit_after>'''Not sure if this is useful -- try training the Tensorizer component.'''
import plac
import spacy
import thinc.extra.datasets
from spacy.util import minibatch, use_gpu
import tqdm


def load_imdb():
    nlp = spacy.blank('en')
    train, dev = thinc.extra.datasets.imdb()
    train_texts, _ = zip(*train)
    dev_texts, _ = zip(*dev)
    nlp.add_pipe(nlp.create_pipe('sentencizer'))
    return list(train_texts), list(dev_texts)


def get_sentences(nlp, texts):
    for doc in nlp.pipe(texts):
        for sent in doc.sents:
            yield sent.text


def prefer_gpu():
    used = spacy.util.use_gpu(0)
    if used is None:
        return False
    else:
        return True

def main(vectors_model):
    use_gpu = prefer_gpu()
    print("Using GPU?", use_gpu)
    print("Load data")
    train_texts, dev_texts = load_imdb()
    print("Load vectors")
    nlp = spacy.load(vectors_model)
    print("Start training")
    nlp.add_pipe(nlp.create_pipe('tagger'))
    tensorizer = nlp.create_pipe('tensorizer')
    nlp.add_pipe(tensorizer)
    optimizer = nlp.begin_training()

    for i in range(10):
        losses = {}
        for i, batch in enumerate(minibatch(tqdm.tqdm(train_texts))):
            docs = [nlp.make_doc(text) for text in batch]
            tensorizer.update(docs, None, losses=losses, sgd=optimizer, drop=0.5)
        print(losses)

if __name__ == '__main__':
    plac.call(main)

{'fraction_non_alphanumeric': 0.12119039082108282, 'fraction_numerical': 0.005736823234134099, 'mean_word_length': 3.1704035874439462, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4538223', 'n_tokens_mistral': 1001, 'n_tokens_neox': 976, 'n_words': 222}