<jupyter_start><jupyter_text>Preprocessing Bert models  installing missing libraries<jupyter_code>!pip install transformers
# general libraries
from tqdm import tqdm
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# bert related libraries
import torch
import tensorflow as tf
from torch.utils.data import DataLoader, TensorDataset
from transformers import AutoTokenizer, AutoModel, AutoModelWithLMHead<jupyter_output><empty_output><jupyter_text>Reading data from google drive<jupyter_code>baseDir = '/content/gdrive/My Drive/Colab Notebooks/AA'
from google.colab import drive
drive.mount('/content/gdrive')
df = pd.read_json(baseDir+'/data/AllDS.json.zip', orient='records', compression='gzip')
df.head(3)
# row index will help simplify the joins/merges, tracking down the original line 
df['row_index']  = np.arange(len(df))
df.head(10)<jupyter_output><empty_output><jupyter_text>Bert Code<jupyter_code>#which bert model should be used for each language.

bert_config ={
    'en' :'bert-base-cased',
    'pt': "neuralmind/bert-large-portuguese-cased",
    'fr':'dbmdz/bert-base-french-europeana-cased',
    'sp': "dccuchile/bert-base-spanish-wwm-cased",
    'it': "dbmdz/bert-base-italian-cased",
    'pl': "dkleczek/bert-base-polish-cased-v1",
    'all': "bert-base-multilingual-cased",
}<jupyter_output><empty_output><jupyter_text>Generating token statistics to make sure 512 tokens is enough.<jupyter_code>for lang in df['language'].unique():
    print("processing %s  - %s" % (lang, bert_config[lang]));

    tokenizer = AutoTokenizer.from_pretrained(bert_config[lang])
    tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t), df.query("language == '%s'" %lang)['text']))
    display(pd.DataFrame({'token_len':[len(t) for t in tokens]}).describe(percentiles=[0.25,0.5,0.75,0.8,0.9,0.99,1]).T)
    
def processDF(df, textToIdFunction):
    """
    returns Bert vector for chunks of 512 tokens
    df: 
    """

    bert_results = {
        'row_index':[],
        'lang': [],
        'vectors': [],
    }
    for lang in df['language'].unique():
        print(f"processing {lang}  - {bert_config[lang]}");

        tokenizer = AutoTokenizer.from_pretrained(bert_config[lang])
        model = AutoModel.from_pretrained(bert_config[lang]).cuda()


        subset_df = df.query("language == '%s'" %lang);
        print(" %d documents found." % len(subset_df));
        texts = subset_df['text']
        row_index = subset_df['row_index']

        #tokenizing
        MAX_TOKENS = 512
        tokens, row_index = textToIdFunction(tokenizer, texts, row_index, MAX_TOKENS)

        tokens_ids = list(map(tokenizer.convert_tokens_to_ids, tokens))
        tokens_ids = tf.keras.preprocessing.sequence.pad_sequences(tokens_ids, maxlen=MAX_TOKENS, truncating='post', padding='post', dtype='int64')

        tensors = torch.tensor(tokens_ids)
        dataloader = DataLoader(TensorDataset(tensors), batch_size=32)


        with torch.no_grad():
            results = []
            for t in tqdm(iter(dataloader)):
                t = t[0]
                result = model(t.cuda())
                result = result[0][range(len(t)), -1, :]
                results.append(result.cpu())
            results = torch.cat(results).numpy()


        bert_results['row_index'].append(row_index);
        bert_results['lang'].append([lang]* len(row_index));
        bert_results['vectors'].append(results);

    return bert_results;
def textToIdCrop(tokenizer, texts, row_index,max_tokens=512):
    tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:(max_tokens-1)], texts))
    return tokens, row_index;<jupyter_output><empty_output><jupyter_text>training bert with first 512 tokens<jupyter_code>bert_results = processDF(df,textToIdCrop)
bert_results2 = pd.DataFrame({
    'row_index': [i for r in bert_results['row_index'] for i in r],
    'BERT_vector': [vv for v in bert_results['vectors'] for vv in v]
})
df_with_bert = pd.merge( df, bert_results2,on='row_index').drop(columns=['row_index'])
df_with_bert.to_json(baseDir+'/data/AllDS_BERT.json.zip', orient='records', compression='gzip')
print(len(df_with_bert))
df_with_bert.head(100)<jupyter_output>30966
{'fraction_non_alphanumeric': 0.1431980906921241, 'fraction_numerical': 0.012649164677804296, 'mean_word_length': 4.2322097378277155, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '441849', 'license': 'CC0-1.0', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1417, 'n_words': 270, 'path': 'bert/00_preprocessing_bert.ipynb', 'repo_name': 'jeleandro/DynamicAuthorshipAttribution', 'size': '4190'}