<commit_before>if __name__ == "__main__":
    from preprocess.data import ReVerbPairs, UNKNOWN_TOKEN
    from word2vec import WORD_EMBEDDING_BIN_FILE, EMBEDDING_SIZE
    from preprocess.data import WordEmbeddingRaw
    from collections import defaultdict
    import pickle as pkl
    import numpy as np
    import os
    import sys
    import logging

    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

    if os.path.exists(WORD_EMBEDDING_BIN_FILE):
        logging.info("Word embedding dictionary file exists, skip")
        sys.exit(0)

    # the word embedding text file use raw word
    src_data = WordEmbeddingRaw()

    logging.info("converting raw embedding text")
    word_emb_hash = {}

    line_num = 1
    for w, emb in src_data:
        sys.stdout.write("\rLoad: %d/%d, %.2f%%" % (line_num, len(src_data), line_num/len(src_data)*100))
        sys.stdout.flush()
        line_num += 1
        word_emb_hash[w] = emb

    #
    #  calculate embedding for UNKNOWN token
    #
    # get token occur time
    token_occur_count = defaultdict(int)
    src_data = ReVerbPairs(usage='train', mode='raw_token', grams=1)
    for line in src_data:
        for i in src_data.sent_indx:
            for token in line[i]:
                token_occur_count[token] += 1

    unknown_emb = np.zeros(EMBEDDING_SIZE, dtype='float32')
    unknown_count = 0
    for token, occur_count in token_occur_count.items():
        if occur_count == 1:
            try:
                unknown_emb += word_emb_hash[token]
                unknown_count += 1
            except KeyError:
                continue
    unknown_emb /= float(unknown_count)
    word_emb_hash[UNKNOWN_TOKEN] = unknown_emb

    sys.stdout.write("\n")
    logging.info("Saving word embedding dictionary")
    with open(WORD_EMBEDDING_BIN_FILE, 'wb') as f:
        pkl.dump(word_emb_hash, f, protocol=4)
<commit_msg>set occurency == 3 as the UNKNOWN token<commit_after>if __name__ == "__main__":
    from preprocess.data import ReVerbPairs, UNKNOWN_TOKEN
    from word2vec import WORD_EMBEDDING_BIN_FILE, EMBEDDING_SIZE
    from preprocess.data import WordEmbeddingRaw
    from collections import defaultdict
    import pickle as pkl
    import numpy as np
    import os
    import sys
    import logging

    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

    if os.path.exists(WORD_EMBEDDING_BIN_FILE):
        logging.info("Word embedding dictionary file exists, skip")
        sys.exit(0)

    # the word embedding text file use raw word
    src_data = WordEmbeddingRaw()

    logging.info("converting raw embedding text")
    word_emb_hash = {}

    line_num = 1
    for w, emb in src_data:
        sys.stdout.write("\rLoad: %d/%d, %.2f%%" % (line_num, len(src_data), line_num/len(src_data)*100))
        sys.stdout.flush()
        line_num += 1
        word_emb_hash[w] = emb
    sys.stdout.write("\n")

    #
    #  calculate embedding for UNKNOWN token
    #
    # get token occur time
    logging.info("calculating embedding for UNKNOWN token")
    token_occur_count = defaultdict(int)
    src_data = ReVerbPairs(usage='train', mode='raw_token', grams=1)
    for line in src_data:
        for i in src_data.sent_indx:
            for token in line[i]:
                token_occur_count[token] += 1

    unknown_emb = np.zeros(EMBEDDING_SIZE, dtype='float32')
    unknown_count = 0
    for token, occur_count in token_occur_count.items():
        if occur_count == 3:
            try:
                # for token only occur once and token have trained embedding
                unknown_emb += word_emb_hash[token]
                unknown_count += 1
            except KeyError:
                continue
    # get average of embedding
    unknown_emb /= float(unknown_count)
    word_emb_hash[UNKNOWN_TOKEN] = unknown_emb

    logging.info("Saving word embedding dictionary")
    with open(WORD_EMBEDDING_BIN_FILE, 'wb') as f:
        pkl.dump(word_emb_hash, f, protocol=4)

{'fraction_non_alphanumeric': 0.09260178748758689, 'fraction_numerical': 0.008192651439920555, 'mean_word_length': 2.7654205607476636, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1601224', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1272, 'n_words': 321}