<jupyter_start><jupyter_text>word2vec definition embeddings<jupyter_code>wiki2seqId = {}
with open("/datadrive/large_data/FosWikiIdToSeqId.txt", "r") as fin:
    for line in fin:
        line = line.strip()
        if line:
            segs = line.split("\t")
            wiki2seqId[segs[0]] = int(segs[1])
tx_id2name = {}
with open("/datadrive/structure_expan/data/MAG_FoS/mag_field_of_studies.terms", "r") as fin:
    for line in fin:
        line = line.strip()
        if line:
            segs = line.split("\t")
            tx_id2name[int(segs[0])] = segs[1]
seqId2embed = {}
with open("/datadrive/large_data/MAG_pretrained_definition_word2vec.tsv", "r") as fin:
    for idx, line in enumerate(fin):
        if idx % 100000 == 0:
            print(idx)
        line = line.strip()
        if line:
            segs = line.split("\t")
            wikiId = segs[0]
            if wikiId in wiki2seqId:
                seqId = wiki2seqId[wikiId]
                seqId2embed[seqId] = segs[1]
tx_id2embed = {}
for tx_id in tx_id2name:
    if tx_id in seqId2embed:
        tx_id2embed[tx_id] = seqId2embed[tx_id]
len(tx_id2name)
len(tx_id2embed)
with open("/datadrive/structure_expan/data/MAG_FoS/mag_field_of_studies.terms.word2vec.def.embed", "w") as fout:
    fout.write(f"{len(tx_id2name)} 250\n")
    for seqId in tx_id2name:
        if seqId in tx_id2embed:
            fout.write(f"{seqId} {tx_id2embed[seqId]}\n")
        else:
            embed_string = " ".join(["-1000" for _ in range(250)])
            fout.write(f"{seqId} {embed_string}\n")
wv = KeyedVectors.load_word2vec_format("/datadrive/structure_expan/data/MAG_FoS/mag_field_of_studies.terms.word2vec.def.embed")
(wv.vectors.sum(axis=1) == -1000 * 250).sum()
wv.vectors.shape
for tx_id in tx_id2embed:
    embed_string = tx_id2embed[tx_id]
    embed_sum = np.array([float(ele) for ele in embed_string.split(" ")]).sum()
    if embed_sum == -1000 * 250:
        print(tx_id, tx_id2name[tx_id])
nx.number_weakly_connected_components(final_fos_taxonomy)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13560975609756099, 'fraction_numerical': 0.03170731707317073, 'mean_word_length': 2.9825242718446603, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '334642', 'license': 'Apache-2.0', 'n_tokens_mistral': 831, 'n_tokens_neox': 782, 'n_words': 134, 'path': 'data_preprocessing/mag-all-fos.ipynb', 'repo_name': 'jinfenglin/TaxoExpan', 'size': '2050'}