<jupyter_start><jupyter_text>Treat BERT as a contextualized embedding generator This notebook will get the necessary folders ready and generate the necessary text files for bert<jupyter_code>%load_ext autoreload
%autoreload

import os
from sys import path
path.append('../..')
from relation_extraction.data import utils
# from bert import extract_features
# from relation_extraction.data import utils
# import h5py
# import numpy as np
# dataset = 'semeval2010' #TODO: update this
preprocessing_type = 'entity_blinding' #TODO: update this; mostly we are only going to be using original
# cuda_device=3 #cuda device for the elmo embeddings generator
dataset = 'ddi'
data_path = '/data/scratch-oc40/geeticka/data/relation_extraction/' + dataset + '/'
def res(path): return os.path.join(data_path, path)
out_path = 'pre-processed/'+ preprocessing_type +'/bert-CLS/'
# if not os.path.exists(res(out_path)):
#     os.makedirs(res(out_path))
# if not os.path.exists(res(out_path + 'input-sentences')):
#     os.makedirs(res(out_path + 'input-sentences'))
border_sizes = [-1] # for i2b2
def get_sentences(filename, border_size):
    data = open(res('pre-processed/'+ preprocessing_type+'/' + filename))
    data = utils.split_data_cut_sentence(data, border_size)
    sentences = data[0]
    return sentences
def write_sentences_to_txt(sentences, filename):
    with open(res(out_path + 'input-sentences/'+filename + '.txt'), 'w') as f:
        for sentence in sentences:
            sentence_string = " ".join(sentence)
            f.write(sentence_string + "\n")
# for border_size in border_sizes:
#     train_sentences = get_sentences('train_'+ preprocessing_type +'.txt', border_size)
#     test_sentences = get_sentences('test_'+ preprocessing_type +'.txt', border_size)
#     write_sentences_to_txt(train_sentences, 'train_'+ preprocessing_type +'_border_' + str(border_size))
#     write_sentences_to_txt(test_sentences, 'test_'+ preprocessing_type +'_border_' + str(border_size))<jupyter_output><empty_output><jupyter_text>Now, proceed to generating the BERT embeddings using the scripts provided in pytorch inside the bert folder where these sentences are located  Now, we would like to read these BERT embeddings generated in the BERT folder and generate the json file without the word piece embeddings<jupyter_code>in_path = 'pre-processed/'+ preprocessing_type +'/bert-CLS/'
out_path = 'pre-processed/' + preprocessing_type + '/bert-tokens/'
if not os.path.exists(res(out_path)):
    os.makedirs(res(out_path))
border_size = -1
train_filename = 'train_' + preprocessing_type + '_border_' + str(border_size) + '.json'
test_filename = 'test_' + preprocessing_type + '_border_' + str(border_size) + '.json'
utils.write_bert_tokens_without_word_pieces(res(in_path + train_filename), res(out_path + train_filename))
utils.write_bert_tokens_without_word_pieces(res(in_path + test_filename), res(out_path + test_filename))
import json
with open(res(out_path + train_filename), 'r', encoding='utf-8') as file:
    length = 0
    for line in file.readlines():
        data = json.loads(line)
        length += 1
length
with open(res(out_path + test_filename), 'r', encoding='utf-8') as file:
    length = 0
    for line in file.readlines():
        data = json.loads(line)
        length += 1
length<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11522262334536702, 'fraction_numerical': 0.005716004813477738, 'mean_word_length': 5.134686346863469, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '37991', 'license': 'MIT', 'n_tokens_mistral': 1084, 'n_tokens_neox': 1064, 'n_words': 277, 'path': 'notebooks/Contextualized Embeddings/Bert Embeddings Generate.ipynb', 'repo_name': 'geetickachauhan/relation-extraction', 'size': '3324'}