<jupyter_start><jupyter_text>Augmentation of less common categories with forward and back translation  Loading the necessary libraries and the data set<jupyter_code>import pandas as pd
#mport deep-translator
from deep_translator import GoogleTranslator #, MyMemoryTranslator
from sklearn.utils import shuffle
import time
import progressbar
import random
# loading data set with comments to be augmented
data = pd.read_csv('data/df_cleaned&added_trans.csv')
# adding a columns for information about augmented or not
#data["augmented"] = 0
#data.clean_comments = data.clean_comments.astype(str)
data.tail()
# For reducing the data set to the less categories; uncomment if necessary
#data_threat = data.query("threat==1 or severe_toxic==1 or identity_hate==1")
#data_threat.head()
# Or if the new data set is in a different file
data_threat = pd.read_csv("data/new_coms.csv")
# adding some missing columns
data_threat["augmented"] = 0
data_threat["clean_comments_without_stop_w"] = "NaN"
data_threat["lem_comments"] = "NaN"
data_threat["stem_comments"] = "NaN"<jupyter_output><empty_output><jupyter_text>Augmentation of comments<jupyter_code>print(data.shape)
index = data_threat.index.tolist()
index_max = data.index.max()
print(data_threat.shape)
print("----------")
batchsize = list(range(0, len(index), int(len(index)/20)))
batchsize.append(data_threat.shape[0])
#for k in tqdm(range(data_threat.shape[0])):
bar = progressbar.ProgressBar(max_value=data_threat.shape[0])
k = 18 # da 85% schon erledigt
for j in range(k, len(batchsize)-1):
    start = time.perf_counter()
    k += 1
    for i in index[batchsize[j]:batchsize[j+1]]:
        inter = data_threat.loc[[i]]
        # if additional dataset, otherwise comment the next line out
        data = data.append(inter, ignore_index=True, verify_integrity=True)
        original= inter.comment_text.iloc[0][:2000]
        #translate to german
        translated = GoogleTranslator(source='english', target='german').translate(original)  
        time.sleep(random.random()*4)
        #and back to english
        translated = GoogleTranslator(source='german', target='english').translate(translated[:2000])
        #Another translator
        #translated = MyMemoryTranslator(source='german', target='english').translate(translated)
        time.sleep(random.random()*4)
        #check if backtranslated not identical to original comment
        if translated != original:
            index_max += 1
            inter.index = [index_max]
            inter.comment_text = translated 
            inter.clean_comments = translated
            inter.id = "NaN"
            inter.clean_comments_without_stop_w = "NaN"
            inter.lem_comments = "NaN"
            inter.stem_comments = "NaN"
            inter["augmented"] = 1
            data = data.append(inter, ignore_index=True, verify_integrity=True)
        bar.update(i)
    end = time.perf_counter()    
    print("=========")
    print(k,":",k*5,"%","5% in ", end-start," s")
    #print(inter)
    data = shuffle(data)
    data.to_csv('data/df_cleaned&added_trans.csv', index=False)
print(data.shape)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12221514958625079, 'fraction_numerical': 0.010502864417568428, 'mean_word_length': 4.077544426494346, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '210411', 'license': 'MIT', 'n_tokens_mistral': 1005, 'n_tokens_neox': 953, 'n_words': 228, 'path': 'modeling/preprocessing/translation_data_augmentation.ipynb', 'repo_name': 'Merlin1674/Capstone-Toxic_Comment_Classification', 'size': '3142'}