<commit_before>
##pre-requisites:
#library(devtools)
#load_all("text2vec")
#install("text2vec")
#build("text2vec")


print("started running trainGloveModel.r")

library("text2vec")

text8_file = "/data/wordvet/wiki.shuffled-norm1-phrase1"

wiki = readLines(text8_file, n = 10000000, warn = FALSE)
# Create iterator over tokens
tokens <- space_tokenizer(wiki)
# Create vocabulary. Terms will be unigrams (simple words).
it = itoken(tokens, progressbar = FALSE)

vocab <- create_vocabulary(it)
vocab <- prune_vocabulary(vocab, doc_proportion_max = 0.3, 
                          max_number_of_terms = 30000L, term_count_min = 5L)
# Use our filtered vocabulary
vectorizer <- vocab_vectorizer(vocab, skip_grams_window=10) 
                               
tcm <- create_tcm(it, vectorizer)

ind <- tcm@x >= 1
tcm@x <- tcm@x[ind]
tcm@i <- tcm@i[ind]
tcm@j <- tcm@j[ind]

glove = GlobalVectors$new(word_vectors_size = 300, vocabulary = vocab, x_max = 10)
glove$fit(tcm, n_iter = 10)

dd <- glove$dump_model()
wordVec = dd$w_i
contextVec = dd$w_j
listVocab = vocab$vocab[,1]

W = cbind(listVocab, wordVec)
fileConn<-file("/data/wordvet/vectorsW.txt")
writeLines(paste(nrow(wordVec), ncol(wordVec)),fileConn )
close(fileConn)
write.table(W, file = "/data/wordvet/vectorsW.txt", append = TRUE, quote = FALSE, sep = " ",
            eol = "\n", dec = ".", row.names = FALSE,
            col.names = FALSE, qmethod = c("escape", "double"),
            fileEncoding = "")

C = cbind(listVocab, contextVec)           
fileConn<-file("/data/wordvet/vectorsC.txt")
writeLines(paste(nrow(contextVec), ncol(contextVec)),fileConn )
close(fileConn)
write.table(C, file = "/data/wordvet/vectorsC.txt", append = TRUE, quote = FALSE, sep = " ",
            eol = "\n", dec = ".", row.names = FALSE,
            col.names = FALSE, qmethod = c("escape", "double"),
            fileEncoding = "")
<commit_msg>Improve glove model<commit_after>
##pre-requisites:
#library(devtools)
#load_all("text2vec")
#install("text2vec")
#build("text2vec")


print("started running trainGloveModel.r")

library("text2vec")

text8_file = "/data/wiki.shuffled-norm1-phrase1"

wiki = readLines(text8_file, n = 10000000, warn = FALSE)
# Create iterator over tokens
tokens <- space_tokenizer(wiki)
# Create vocabulary. Terms will be unigrams (simple words).
it = itoken(tokens, progressbar = FALSE)

vocab <- create_vocabulary(it)
vocab <- prune_vocabulary(vocab, doc_proportion_max = 0.3, 
                          max_number_of_terms = 30000L, term_count_min = 5L)
# Use our filtered vocabulary
vectorizer <- vocab_vectorizer(vocab, skip_grams_window=10) 
                               
tcm <- create_tcm(it, vectorizer)

ind <- tcm@x >= 1
tcm@x <- tcm@x[ind]
tcm@i <- tcm@i[ind]
tcm@j <- tcm@j[ind]

glove = GlobalVectors$new(word_vectors_size = 300, vocabulary = vocab, x_max = 10)
glove$fit(tcm, n_iter = 10)

dd <- glove$dump_model()
wordVec = dd$w_i
contextVec = dd$w_j
listVocab = vocab$vocab[,1]

W = cbind(listVocab, wordVec)
fileConn<-file("/data/model_200_10/vectorsW.txt")
writeLines(paste(nrow(wordVec), ncol(wordVec)),fileConn )
close(fileConn)
write.table(W, file = "/data/model_200_10/vectorsW.txt", append = TRUE, quote = FALSE, sep = " ",
            eol = "\n", dec = ".", row.names = FALSE,
            col.names = FALSE, qmethod = c("escape", "double"),
            fileEncoding = "")

C = cbind(listVocab, contextVec)           
fileConn<-file("/data/model_200_10/vectorsC.txt")
writeLines(paste(nrow(contextVec), ncol(contextVec)),fileConn )
close(fileConn)
write.table(C, file = "/data/model_200_10/vectorsC.txt", append = TRUE, quote = FALSE, sep = " ",
            eol = "\n", dec = ".", row.names = FALSE,
            col.names = FALSE, qmethod = c("escape", "double"),
            fileEncoding = "")

{'fraction_non_alphanumeric': 0.15346011621764394, 'fraction_numerical': 0.02377179080824089, 'mean_word_length': 3.7101990049751246, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '576292', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1354, 'n_words': 272}