<jupyter_start><jupyter_text>Loading models, computing meta-embeddings<jupyter_code># MorphoSeg models
ms_input = remove_dummy_token(KeyedVectors.load_word2vec_format('models/morphoseg/model.txt'))
ms_output = remove_dummy_token(KeyedVectors.load_word2vec_format('models/morphoseg/context.txt'))

# FastText models
ft_input = remove_dummy_token(KeyedVectors.load_word2vec_format('models/fasttext/model.txt'))
ft_output = remove_dummy_token(KeyedVectors.load_word2vec_format('models/fasttext/context.txt'))
def merge(model1, model2):
    model_concat = concat(model1, model2)
    model_reduced = dim_reduce(model_concat, 300)
    del model_concat
    return model_reduced
# two-way merges
ms_merged = merge(ms_input, ms_output)
ft_merged = merge(ft_input, ft_output)
# triple merge
triple_merged = merge(ms_merged, ft_merged)
triple_merged.save_word2vec_format('models/triplemerge.txt')<jupyter_output><empty_output><jupyter_text>Evaluation on test set MorphoSeg, input<jupyter_code>eval_on_analogy_dset(ms_input, testsub, 10, keys)
eval_on_analogy_dset(ms_input, testnonsub, 10, keys)<jupyter_output><empty_output><jupyter_text>MorphoSeg, output<jupyter_code>eval_on_analogy_dset(ms_output, testsub, 10, keys)
eval_on_analogy_dset(ms_output, testnonsub, 10, keys)<jupyter_output><empty_output><jupyter_text>FastText, input<jupyter_code>eval_on_analogy_dset(ft_input, testsub, 10, keys)
eval_on_analogy_dset(ft_input, testnonsub, 10, keys)<jupyter_output><empty_output><jupyter_text>FastText, output<jupyter_code>eval_on_analogy_dset(ft_output, testsub, 10, keys)
eval_on_analogy_dset(ft_output, testnonsub, 10, keys)<jupyter_output><empty_output><jupyter_text>MorphoSeg, merged<jupyter_code>eval_on_analogy_dset(ms_merged, testsub, 10, keys)
eval_on_analogy_dset(ms_merged, testnonsub, 10, keys)<jupyter_output><empty_output><jupyter_text>FastText, merged<jupyter_code>eval_on_analogy_dset(ft_merged, testsub, 10, keys)
eval_on_analogy_dset(ft_merged, testnonsub, 10, keys)<jupyter_output><empty_output><jupyter_text>Triple merged<jupyter_code>eval_on_analogy_dset(triple_merged, testsub, 10, keys)
eval_on_analogy_dset(triple_merged, testnonsub, 10, keys)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10247026532479414, 'fraction_numerical': 0.018298261665141813, 'mean_word_length': 13.294117647058824, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '554022', 'license': 'MIT', 'n_tokens_mistral': 917, 'n_tokens_neox': 861, 'n_words': 103, 'path': '.ipynb_checkpoints/Meta-Embeddings and Evaluation on Analogy Dataset-checkpoint.ipynb', 'repo_name': 'arjun-sai-krishnan/tamil-morpho-embeddings', 'size': '2186'}