<jupyter_start><jupyter_text>Load data<jupyter_code>wordembedding = np.load('./origin_data/vec.npy')

test_word = np.load('./data/testall_word.npy')<jupyter_output><empty_output><jupyter_text>Add entity tag<jupyter_code>def _add_entity_tag(row):
    token_sen = row['sen'].split()
    out_token_sen = copy.deepcopy(token_sen)
    update_list_e1 = []
    update_list_e2 = []
    for i, j in enumerate(token_sen):
        if j == row['e1']:
            tmp = i+len(update_list_e1)+len(update_list_e2)
            out_token_sen.insert(tmp, '<e1>')
            out_token_sen.insert(tmp+2, '</e1>')
            
            update_list_e1.append(tmp)
            update_list_e1.append(tmp+2)
        if j == row['e2']:
            tmp = i+len(update_list_e1)+len(update_list_e2)
            update_list_e2.append(tmp)
            update_list_e2.append(tmp+2)
            out_token_sen.insert(tmp, '<e2>')
            out_token_sen.insert(tmp+2, '</e2>')
    temp_row = copy.deepcopy(row)
    temp_row['sen'] = ' '.join(out_token_sen)
    return ' '.join(out_token_sen), temp_row
# Function verification
print(_add_entity_tag(full_data.iloc[0]))
def prepare_bert_data(dataPath):
    full_data = pd.read_csv(dataPath, header=None, sep='\t').iloc[:, 2:]
    full_data.columns = ['e1', 'e2', 'rel', 'sen']
    tagged_sen = []
    row_list = []
    with tqdm(total=len(full_data)) as pbar:
        for _, row in full_data.iterrows():
            temp_sen, temp_row = _add_entity_tag(row)
            tagged_sen.append(temp_sen)
            if len(temp_row['sen'].split())<512:
                row_list.append(temp_row)
            pbar.update(1)
    full_data.drop(columns='sen')
    full_data['seq'] = tagged_sen
    full_data = full_data.fillna(value='UNK')
    
    cleaned_df = pd.DataFrame(row_list)
    cleaned_df = cleaned_df.fillna(value='UNK')
    cleaned_df = cleaned_df.iloc[:, 2:]
    cleaned_df.to_csv(dataPath[:-4]+'_filtered.txt', index=False, sep='\t')
    full_data.to_csv(dataPath[:-4]+'_bert.txt', index=False, sep='\t')
    
def _clean_text(dataPath):
    output = []
    with open(dataPath, 'r') as origin_file:
        baselen = 0
        n_line = 1

        for line in origin_file.readlines():
            line = line.strip()
            token = line.split('\t')
            if baselen == 0:
                baselen = len(token)
            else:
                if len(token) != baselen:
                    print(token)
                    print(n_line)
            n_line += 1
            temp = '\t'.join(token[:6])+'\n'
            output.append(temp)
    os.rename(dataPath, dataPath[:-4]+'_original.txt')
    with open(dataPath, 'w') as outfile:
        outfile.writelines(output)
prepare_bert_data('./origin_data/test.txt')

tagged_sen = []
with tqdm(total=len(full_data)) as pbar:
    for _, row in full_data.iterrows():
        tagged_sen.append(_add_entity_tag(row))
        pbar.update(1)
print(full_data.iloc[0]['tagged'])
def convert_filter(dataPath):
    df = pd.read_csv(dataPath, sep='\t', header=None)
    df.columns=['labels', 'text']
#    df.to_json(dataPath[:-3]+'json', orient='records')
    df.to_json(dataPath[:-3]+'json')
convert_filter('./origin_data/train_filtered.txt')
def filter_long(path):
    df = pd.read_csv(path, header=None, sep='\t')
    temp = []
    for _, row in df.iterrows():
        token = row.iloc[-1].split()
        if len(token)<480:
            temp.append(row)
        else:
            print(len(token))
    print(len(temp))
    print(len(df))
filter_long('./origin_data/test.txt')
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
def convert_label(path):
    
    df = pd.read_csv(path, header=None, sep='\t')
    if not hasattr(le, 'classes_'):
        le.fit(df.iloc[:, 0])
    df.iloc[:, 0] = le.transform(df.iloc[:, 0])
    
    df.to_csv(path, header=False, index=False, sep='\t')
convert_label('./origin_data/train_filtered.txt')
print(hasattr(le, 'classes_'))
convert_label('./origin_data/test_filtered.txt')
from transformers import BertTokenizer
pretrain_model = "bert-base-uncased"
additional_special_tokens = ['<e1>', '</e1>', '<e2>', '</e2>']
tokenizer = BertTokenizer.from_pretrained(pretrain_model, do_lower_case=True, additional_special_tokens = additional_special_tokens)

def tokenization(tokenizer, row):
    '''
    Tokenize the sentences from filter data
    '''
    sentence = '[CLS] '+row.iloc[-1]+' [SEP]'
    token = tokenizer.tokenize(sentence)
    return len(token)

def bert_token_filter(path):
    original_data = pd.read_csv(path, sep='\t', header=None)
    temp_row = []
    for _, row in tqdm(original_data.iterrows()):
        token_len = tokenization(tokenizer, row)
        if token_len>512:
            print(token_len)
        else:
            row.iloc[-1] = '[CLS] '+row.iloc[-1]+' [SEP]'
            temp_row.append(row)
    out_df = pd.DataFrame(temp_row)
    out_df.to_csv(path[:-4]+'_bf.txt', header=False, index=False, sep='\t')
data_path = [
    './origin_data/train_filtered.txt',
    './origin_data/test_filtered.txt'
]
for i in data_path:
    bert_token_filter(i)
original_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1560614905623662, 'fraction_numerical': 0.011480832846857365, 'mean_word_length': 3.0282131661442007, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '551640', 'license': 'Apache-2.0', 'n_tokens_mistral': 1869, 'n_tokens_neox': 1842, 'n_words': 290, 'path': 'prepare.ipynb', 'repo_name': 'Molin-L/RLRC', 'size': '5139'}