<jupyter_start><jupyter_text>Training the model<jupyter_code>import datareader as dr
import word2vec as wv
import time
import util
ts = time.time()
file_path = "./data/pt96.txt"
my_data = dr.DataReader(file_path)
my_data.get_data()
word2index = my_data.word2index
index2word = my_data.index2word
config = wv.Config()
my_model = wv.SkipGramModel(config)
embeddings = wv.run_training(my_model, my_data)
te = time.time()
#! tensorboard ..
#! tensorboard  --logdir=./processed
print("Duration of the training = {}".format(te-ts))<jupyter_output>Duration of the training = 612.1953608989716<jupyter_text>Evaluating the model<jupyter_code>eval_path = "./evaluation/questions-words-ptbr.txt"
score, report = util.score(index2word, word2index, embeddings, eval_path,verbose=False,raw=True)

print("Overall score ={}".format(score))
print("Detailed score:")
for result in report:
    print(result)<jupyter_output>Overall score =0.007391763463569166
Detailed score:
capital-common-countries: (0/306)
capital-world: (0/1155)
currency: (0/106)
city-in-state: (0/1169)
family: (40/342)
gram1-adjective-to-adverb: (0/552)
gram2-opposite: (0/182)
gram3-comparative: (7/30)
gram4-superlative: (3/20)
gram5-present-participle: (2/702)
gram6-nationality-adjective: (0/412)
gram7-past-tense: (0/1056)
gram8-plural: (0/992)
gram9-plural-verbs: (4/552)<jupyter_text>Comparing with gensim gensim.models.Word2Vec(sentences=None, size=100, alpha=0.025, window=5, min_count=5, max_vocab_size=None, sample=0.001, seed=1, workers=3, min_alpha=0.0001, sg=0, hs=0, negative=5, cbow_mean=1, hashfxn=, iter=5, null_word=0, trim_rule=None, sorted_vocab=1, batch_words=10000)<jupyter_code>import gensim, logging
import os
import inspect
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)


class MySentences(object):
    def __init__(self, dirname):
        self.dirname = dirname
 
    def __iter__(self):
        for fname in os.listdir(self.dirname):
            for line in open(os.path.join(self.dirname, fname)):
                yield line.split()

file_path = "./data/gensim"
sentences = MySentences(file_path)
model = gensim.models.Word2Vec(sentences)
model.accuracy(eval_path)<jupyter_output>2017-03-25 12:39:42,123 : INFO : precomputing L2-norms of word weight vectors
2017-03-25 12:39:42,552 : INFO : capital-common-countries: 8.9% (8/90)
2017-03-25 12:39:43,247 : INFO : capital-world: 2.3% (4/173)
2017-03-25 12:39:43,477 : INFO : currency: 0.0% (0/54)
2017-03-25 12:39:44,294 : INFO : city-in-state: 0.5% (1/208)
2017-03-25 12:39:45,458 : INFO : family: 43.1% (132/306)
2017-03-25 12:39:46,922 : INFO : gram1-adjective-to-adverb: 1.3% (5/380)
2017-03-25 12:39:47,297 : INFO : gram2-opposite: 3.3% (3/90)
2017-03-25 12:39:47,431 : INFO : gram3-comparative: 40.0% (12/30)
2017-03-25 12:39:47,477 : INFO : gram4-superlative: 66.7% (4/6)
2017-03-25 12:39:49,240 : INFO : gram5-present-participle: 13.2% (61/462)
2017-03-25 12:39:52,175 : INFO : gram6-nationality-adjective: 5.0% (37/739)
2017-03-25 12:39:54,342 : INFO : gram7-past-tense: 24.5% (124/506)
2017-03-25 12:39:56,190 : INFO : gram8-plural: 6.6% (25/380)
2017-03-25 12:39:58,109 : INFO : gram9-plural-verbs: 9.5% (29/306)
2017-03-[...]
{'fraction_non_alphanumeric': 0.16521468574984444, 'fraction_numerical': 0.15805849408836342, 'mean_word_length': 5.031894934333959, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '284127', 'license': 'MIT', 'n_tokens_mistral': 1630, 'n_tokens_neox': 1320, 'n_words': 249, 'path': 'src/Script_presentation.ipynb', 'repo_name': 'LIAMF-USP/Word2vec-pt', 'size': '3214'}