<jupyter_start><jupyter_text>doc2vecを用いた単語の特徴ベクトル学習 (テストラン: 1回反復)<jupyter_code>import gensim
import os
import collections
import random
import csv
import sys
csv.field_size_limit(sys.maxsize)
corpusfile = 'novel2vec.txt'
modelfile = 'novel2vec.model'
def read_corpus():
    with open(corpusfile, newline='', encoding='utf-8') as csvfile:
        novels = csv.reader(csvfile)
        for novel in novels:
            novel_id, tokens_text = novel
            tokens = gensim.utils.to_unicode(tokens_text).split()
            yield gensim.models.doc2vec.TaggedDocument(tokens, [novel_id])
corpus = list(read_corpus())
model = gensim.models.doc2vec.Doc2Vec(size=100, min_count=5, iter=1, workers=36)
model.build_vocab(corpus)
import resource
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
%time model.train(corpus)
model.save('novel2vec-iter1.model')
ranks = []
second_ranks = []
for doc_id in range(len(corpus)):
    inferred_vector = model.infer_vector(corpus[doc_id].words)
    sims = model.docvecs.most_similar([inferred_vector], topn=len(model.docvecs))
    rank = [docid for docid, sim in sims].index(corpus[doc_id].tags[0])
    ranks.append(rank)
    second_ranks.append(sims[1])
collections.Counter(ranks)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1229903536977492, 'fraction_numerical': 0.01527331189710611, 'mean_word_length': 5.256281407035176, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '204895', 'license': 'MIT', 'n_tokens_mistral': 489, 'n_tokens_neox': 481, 'n_words': 76, 'path': 'doc2vec-train-iter1.ipynb', 'repo_name': 'pixiv/pixivnovel2vec', 'size': '1244'}