<commit_before>import argparse
from codecs import open
import logging
import os

from gensim.models import Word2Vec

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Prepare model for Gaussian LDA")
    parser.add_argument("--vocabulary", type=str)
    parser.add_argument("--embedding-model", type=str)
    args = parser.parse_args()

    word2vec = Word2Vec.load_word2vec_format(args.embedding_model, binary=True)
    vocab_name = os.path.basename(args.vocabulary)
    embedding_name = os.path.basename(args.embedding_model)
    with open(args.embedding_model + "." + vocab_name + ".model", "w", encoding="utf-8") as output:
        with open(args.vocabulary, "r", encoding="utf-8") as f:
            for line in f:
                word = line.rstrip()
                try:
                    output.write(" ".join(map(str, word2vec[word])))
                except KeyError:
                    print word
                    output.write(" ".join(map(str, word2vec["this"])))
                output.write("\n")<commit_msg>Try different replacement word in gaussian_lda_preprocessing.<commit_after>import argparse
from codecs import open
import logging
import os

from gensim.models import Word2Vec

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Prepare model for Gaussian LDA")
    parser.add_argument("--vocabulary", type=str)
    parser.add_argument("--embedding-model", type=str)
    args = parser.parse_args()

    word2vec = Word2Vec.load_word2vec_format(args.embedding_model, binary=True)
    vocab_name = os.path.basename(args.vocabulary)
    embedding_name = os.path.basename(args.embedding_model)
    with open(args.embedding_model + "." + vocab_name + ".model", "w", encoding="utf-8") as output:
        with open(args.vocabulary, "r", encoding="utf-8") as f:
            for line in f:
                word = line.rstrip()
                try:
                    output.write(" ".join(map(str, word2vec[word])))
                except KeyError:
                    print word
                    output.write(" ".join(map(str, word2vec["house"])))
                output.write("\n")
{'fraction_non_alphanumeric': 0.13065976714100905, 'fraction_numerical': 0.0068995256576110395, 'mean_word_length': 2.8602329450915143, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1217045', 'n_tokens_mistral': 729, 'n_tokens_neox': 714, 'n_words': 148}