<jupyter_start><jupyter_text>Read the data<jupyter_code>classes = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train_data = read_csv("../data/train.csv")
train_data.head()
train_data.fillna('NULL', inplace= True)
train_data['profane'] = train_data[classes].any(axis = 1).astype(int)
y = train_data['profane'].values<jupyter_output><empty_output><jupyter_text>Get word frequency table<jupyter_code># Get the Word Frequency table
tmp = train_data['comment_text'].str.cat(sep=' ')
words = tmp.split()
word_dist_dict = nltk.FreqDist(words)
word_dist_dict_most_common = word_dist_dict.most_common()
word_dist_dict_df = DataFrame(word_dist_dict_most_common, columns=['raw_word', 'freq'])
word_dist_dict_df.to_csv('../data/word_dist_train.csv', index=False)
pickle.dump(word_dist_dict, open('../data/word_dist_train.pkl', 'wb'))
pp_steps = [
            ['raw'],  
            ['convert_to_lower'],
            ['remove_whitespaces'], 
            ['remove_leaky'], 
            ['trim_words_len'],
            ['strip_non_printable_chars'],
            ['replace_abbreviation_words'],
            ['replace_acronyms'],
            ['remove_stopwords'],
            ['remove_rare_words'],
            ['remove_non_alphanumeric'],
            ['remove_non_alphabet_words'],
            ['remove_words_containing_non_alphabets'],
            ['black_listed_words_regex_mapping'],
            ['check_if_proper_name_place_or_ethnicity'],
            ['replace_profane_words_using_fuzzy'],
            ['replace_common_words_using_fuzzy'],
            ['lemmatize_english_words'],
            ['stemming_english_words'],
            ['extract_info_from_url'],
       
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars', 'remove_leaky'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars',  'replace_abbreviation_words', 'replace_acronyms'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars',  'remove_stopwords'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars',  'remove_rare_words'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars', 'replace_abbreviation_words', 'replace_acronyms', 
#      'remove_stopwords', 'remove_rare_words'], 
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars', 'remove_non_alphanumeric'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars', 'remove_non_alphabet_words'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars', 'remove_non_alphanumeric', 
#      'remove_non_alphabet_words'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars','replace_abbreviation_words', 'replace_acronyms',
#      'black_listed_words_regex_mapping'], 
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars','replace_abbreviation_words', 'replace_acronyms',
#      'black_listed_words_regex_mapping', 'replace_profane_words_using_fuzzy', 
#      'replace_common_words_using_fuzzy'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars','replace_abbreviation_words',  'replace_acronyms', 
#      'remove_leaky',  'extract_info_from_url',  'black_listed_words_regex_mapping', 
#      'replace_profane_words_using_fuzzy', 'replace_common_words_using_fuzzy', 
#      'check_if_proper_name_place_or_ethnicity'],
#     ['convert_to_lower', 'remove_whitespaces', 'trim_words_len', 
#      'strip_non_printable_chars', 'replace_abbreviation_words',  'replace_acronyms', 
#      'remove_leaky',  'remove_non_alphabet_words', 'remove_stopwords', 
#      'stemming_english_words'],
#     ['convert_to_lower', 'lemmatize_english_words'],
#     ['convert_to_lower','remove_non_alphabet_words', 
#      'black_listed_words_regex_mapping',  'replace_common_words_using_fuzzy', 
#      'stemming_english_words']
    ]
maxlen = 30

def get_corresponding_mapping(word_dist_dict_most_common, op_type):
    print(op_type)
    if op_type == 'raw':
        new_dict = dict([(x[0], x[0]) for x in word_dist_dict_most_common])
    elif op_type ==  'convert_to_lower': # remove white spaces 
        new_dict = convert_to_lower_from_dict(word_dist_dict_most_common)
    elif op_type ==  'remove_whitespaces': # remove white spaces 
        new_dict = remove_white_spaces_from_dict(word_dist_dict_most_common)
    elif op_type == 'remove_leaky':
        new_dict = remove_leaky_information_from_dict(word_dist_dict_most_common)
    elif op_type == 'extract_info_from_url':
        new_dict = extract_info_from_url(word_dist_dict_most_common, extractor)
    elif op_type == 'trim_words_len':
        new_dict = trim_words_len(word_dist_dict_most_common, maxlen)
    elif op_type == 'replace_abbreviation_words':
        new_dict = replace_abbreviation_words_from_dict(word_dist_dict_most_common, abbrev_words_dict)
    elif op_type == 'strip_non_printable_chars':
        new_dict = strip_non_printable_chars_from_dict(word_dist_dict_most_common)
    elif op_type == 'replace_acronyms':
        new_dict = replace_acronyms_from_dict(word_dist_dict_most_common, acronyms_dict, proper_words_with_profane_dict)
    elif op_type == 'remove_stopwords':
        new_dict = remove_stopwords_from_dict(word_dist_dict_most_common, stop_words_dict)
    elif op_type == 'remove_rare_words':
        new_dict = remove_rare_words_from_dict(word_dist_dict_most_common, word_dist_dict, rare_word_thresh)
    elif op_type == 'remove_non_alphanumeric':
        new_dict = remove_non_alphanumeric_from_dict(word_dist_dict_most_common)    
    elif op_type == 'remove_non_alphabet_words':
        new_dict = remove_non_alphabet_words(word_dist_dict_most_common)    
    elif op_type == 'remove_words_containing_non_alphabets':
        new_dict = remove_words_containing_non_alphabets_from_dict(word_dist_dict_most_common)    
    elif op_type == 'black_listed_words_regex_mapping':
        new_dict = black_listed_words_regex_mapping_from_dict(word_dist_dict_most_common, all_badlist, profane_list_map, extreme_profane)
    elif op_type == 'replace_profane_words_using_fuzzy':
        new_dict = replace_profane_words_using_fuzzy(word_dist_dict_most_common, proper_words_dict, extreme_profane, profane_list_map, badlist_common)
    elif op_type == 'check_if_proper_name_place_or_ethnicity':
        new_dict = check_if_proper_name_place_or_ethnicity_from_dict(word_dist_dict_most_common, proper_words_dict, citynames, countries, nationalities, \
                                              ethnicities, person_names_dict)
    elif op_type == 'replace_common_words_using_fuzzy':
        new_dict = replace_common_words_using_fuzzy(word_dist_dict_most_common, word_dist_dict_most_common, wordnet_lemmatizer, proper_words_dict)
    elif op_type == 'lemmatize_english_words':
        new_dict = lemmatize_english_words(word_dist_dict_most_common, wordnet_lemmatizer)
    elif op_type == 'stemming_english_words':
        new_dict = stemming_english_words(word_dist_dict_most_common, stemmer)
    else:
        print ("Error......")
        new_dict = 'Error .......'
    return new_dict

def get_new_distribution(X):
    tmp = X.str.cat(sep=' ')
    words = tmp.split()
    w_d_dict = nltk.FreqDist(words) #Word distribution dict
    w_d_most_common = w_d_dict.most_common()
    return w_d_most_common

# def get_corresponding_mapping_multiple(word_dist_dict_most_common, op_types):
#     curr_list = word_dist_dict_most_common.copy()
#     new_dict = {}
#     for op_type in op_types:
#         print (op_type)
#         tmp_map = get_corresponding_mapping(curr_list, op_type)
#         if not new_dict:
#             new_dict = tmp_map
#         else: 
#             new_dict = update_dict_with_next_level_val(new_dict, tmp_map)
#         curr_list = [(new_dict[x], 1) for x in new_dict]
#     return new_dict

def get_corresponding_mapping_multiple(word_dist_dict_most_common, op_types, train_data):
    curr_word_dist_dict_most_common = word_dist_dict_most_common.copy()
    comment_data = train_data['comment_text']
    for op_type in op_types:
        print (op_type)
        new_mapped_dict = get_corresponding_mapping(curr_word_dist_dict_most_common, op_type)
        comment_data = comment_data.apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
        curr_word_dist_dict_most_common = get_new_distribution(comment_data)
    return comment_data
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
stop_words_dict = dict(zip(stop_words, [1] * len(stop_words)))<jupyter_output><empty_output><jupyter_text>Calling NB SVM algorithm<jupyter_code>combined_results = pickle.load(open("../data/results_nbsvm.pkl", "rb"))
#Add the preprocessing step here 
import pickle
# combined_results = {} #Uncomment 
for pp_step in pp_steps:
    if len(pp_step) == 1:
        new_mapped_dict = get_corresponding_mapping(word_dist_dict_most_common, pp_step[0])
        X = train_data['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
    else:
        X = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_step, train_data)
    vocab = set(' '.join(X.values).split())
    vectorizer = TfidfVectorizer(ngram_range=(1,4), min_df=1, vocabulary= vocab)
    results = []
    k_fold_num = 0
    cv = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
    for train_index, test_index in cv.split(X, y):
        print ("\n\n *****Processing fold ", k_fold_num, " of ", cv.n_splits, " ......")
        X_train_data, X_test_data = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]
        X_train = vectorizer.fit_transform(X_train_data)
        X_test = vectorizer.transform(X_test_data)
        res = call_NB_SVM_algorithm(X_train, y_train, X_test, y_test)
        results.append(res)
        k_fold_num += 1
#         if k_fold_num ==2:
#             break
    scores = extract_combined_results(results)
    combined_results[' '.join(pp_step)] = scores
    pickle.dump(combined_results, open("../data/results_nbsvm.pkl", "wb"))<jupyter_output><empty_output><jupyter_text>Calling Logistic regression<jupyter_code>combined_results = pickle.load(open("../data/results_logit.pkl", "rb"))
#Add the preprocessing step here 
# combined_results = {} #Uncomment 
for pp_step in pp_steps:
    if len(pp_step) == 1:
        new_mapped_dict = get_corresponding_mapping(word_dist_dict_most_common, pp_step[0])
        X = train_data['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
    else:
        X = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_step, train_data)
    vocab = set(' '.join(X.values).split())
    vectorizer = TfidfVectorizer(ngram_range=(1,4), min_df=1, vocabulary= vocab)
    results = []
    k_fold_num = 0
    cv = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
    for train_index, test_index in cv.split(X, y):
        print ("\n\n *****Processing fold ", k_fold_num, " of ", cv.n_splits, " ......")
        X_train_data, X_test_data = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]
        X_train = vectorizer.fit_transform(X_train_data)
        X_test = vectorizer.transform(X_test_data)
        res = call_logreg_algorithm(X_train, y_train, X_test, y_test)
        results.append(res)
        k_fold_num += 1

#         if k_fold_num ==1:
#             break
    scores = extract_combined_results(results)
    combined_results[' '.join(pp_step)] = scores
    pickle.dump(combined_results, open("../data/results_logit.pkl", "wb"))<jupyter_output>convert_to_lower
convert_to_lower
remove_whitespaces
remove_whitespaces
trim_words_len
trim_words_len


 *****Processing fold  0  of  10  ......


 *****Processing fold  1  of  10  ......


 *****Processing fold  2  of  10  ......


 *****Processing fold  3  of  10  ......


 *****Processing fold  4  of  10  ......


 *****Processing fold  5  of  10  ......


 *****Processing fold  6  of  10  ......


 *****Processing fold  7  of  10  ......


 *****Processing fold  8  of  10  ......


 *****Processing fold  9  of  10  ......
F1-score:  0.7351133951513797
Log Loss:  1.5395094535271634
Accuracy:  0.9554267452061662
AUC score:  0.9715448633514437
Num of comments missclassified:  7113
[[142597    753]
 [  6360   9870]]
             precision    recall  f1-score   support

          0    0.95730   0.99475   0.97567    143350
          1    0.92912   0.60813   0.73511     16230

avg / total    0.95444   0.95543   0.95120    159580

convert_to_lower
convert_to_lower
remove_whitespaces
remove[...]<jupyter_text>Call XGBoost Algorithm<jupyter_code>combined_results = pickle.load(open("../data/results_xgboost.pkl", "rb"))
len(combined_results.keys())
#Add the preprocessing step here 
# combined_results = {} #Uncomment 
for pp_step in pp_steps[20:]:
    if len(pp_step) == 1:
        new_mapped_dict = get_corresponding_mapping(word_dist_dict_most_common, pp_step[0])
        X = train_data['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
    else:
        X = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_step, train_data)
    vocab = set(' '.join(X.values).split())
    vectorizer = TfidfVectorizer(ngram_range=(1,4), min_df=1, vocabulary= vocab)
    results = []
    k_fold_num = 0
    cv = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
    for train_index, test_index in cv.split(X, y):
        print ("\n\n *****Processing fold ", k_fold_num, " of ", cv.n_splits, " ......")
        X_train_data, X_test_data = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]
        X_train = vectorizer.fit_transform(X_train_data)
        X_test = vectorizer.transform(X_test_data)
        res = call_xgboost_algorithm(xgb, vectorizer, X_train, y_train, X_test, y_test)
        results.append(res)
        k_fold_num += 1

#         if k_fold_num ==1:
#             break
    scores = extract_combined_results(results)
    combined_results[' '.join(pp_step)] = scores
    pickle.dump(combined_results, open("../data/results_xgboost.pkl", "wb"))<jupyter_output>convert_to_lower
convert_to_lower
remove_whitespaces
remove_whitespaces
trim_words_len
trim_words_len


 *****Processing fold  0  of  10  ......


 *****Processing fold  1  of  10  ......


 *****Processing fold  2  of  10  ......


 *****Processing fold  3  of  10  ......


 *****Processing fold  4  of  10  ......


 *****Processing fold  5  of  10  ......


 *****Processing fold  6  of  10  ......


 *****Processing fold  7  of  10  ......


 *****Processing fold  8  of  10  ......


 *****Processing fold  9  of  10  ......
F1-score:  0.5734956461465147
Log Loss:  2.1308086271067452
Accuracy:  0.9383068053640807
AUC score:  0.919502285875283
Num of comments missclassified:  9845
[[143116    234]
 [  9611   6619]]
             precision    recall  f1-score   support

          0    0.93707   0.99837   0.96675    143350
          1    0.96585   0.40783   0.57350     16230

avg / total    0.94000   0.93831   0.92675    159580

convert_to_lower
convert_to_lower
remove_whitespaces
remove_[...]<jupyter_text>Fasttext Algorithm<jupyter_code>combined_results = pickle.load(open("../data/individual_fasttext.pkl", "rb"))
len(combined_results.keys())
#combined_results = {}
for pp_step in pp_steps:
    if len(pp_step) == 1:
        new_mapped_dict = get_corresponding_mapping(word_dist_dict_most_common, pp_step[0])
        X = train_data['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
    else:
        X = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_step, train_data)
    results = []
    k_fold_num = 0
    cv = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
    for train_index, test_index in cv.split(X, y):
        print ("\n\n *****Processing fold ", k_fold_num, " of ", cv.n_splits, " ......")
        X_train_data, X_test_data = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]
        texts_train = X_train_data.values
        texts_test  = X_test_data.values
        train = X.copy()
        X_train, X_test, embedding_matrix = preprocess_data_for_fasttext(texts_train, texts_test, train)
        res = call_fasttext_algorithm(X_train, y_train, X_test, y_test, embedding_matrix)
        results.append(res)
        k_fold_num += 1

#         if k_fold_num ==1:
#             break
    scores = extract_combined_results(results)
    combined_results[' '.join(pp_step)] = scores
    pickle.dump(combined_results, open("../data/individual_fasttext_part.pkl", "wb"))<jupyter_output>convert_to_lower
convert_to_lower
remove_whitespaces
remove_whitespaces
trim_words_len
trim_words_len
strip_non_printable_chars
strip_non_printable_chars
replace_abbreviation_words
replace_abbreviation_words
replace_acronyms
replace_acronyms
remove_leaky
remove_leaky
remove_non_alphabet_words
remove_non_alphabet_words
remove_stopwords
remove_stopwords
stemming_english_words
stemming_english_words
Done  50000
Done  100000
Done  150000
Done  200000


 *****Processing fold  0  of  10  ......
Epoch 1/3
143613/143613 [==============================] - 197s 1ms/step - loss: 0.1230 - acc: 0.9549
Epoch 2/3
143613/143613 [==============================] - 176s 1ms/step - loss: 0.1041 - acc: 0.9611
Epoch 3/3
143613/143613 [==============================] - 178s 1ms/step - loss: 0.0991 - acc: 0.9629


 *****Processing fold  1  of  10  ......
Epoch 1/3
143613/143613 [==============================] - 198s 1ms/step - loss: 0.1272 - acc: 0.9536
Epoch 2/3
143613/143613 [==============================[...]<jupyter_text>Greedy to select the best transformation sequence<jupyter_code>#Add the preprocessing step here
def call_logit_to_recursive_transformation_Addition(pp_steps, word_dist_dict_most_common,train_data, algo):
    combined_results = {} #Uncomment 
    for pp_step in pp_steps:
        if len(pp_step) == 1:
            new_mapped_dict = get_corresponding_mapping(word_dist_dict_most_common, pp_step[0])
        else:
            new_mapped_dict = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_step, train_data)
        X = train_data['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
        vocab = set(' '.join(X.values).split())
        vectorizer = TfidfVectorizer(ngram_range=(1,4), min_df=1, vocabulary= vocab)
        results = []
        k_fold_num = 0
        cv = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
        for train_index, test_index in cv.split(X, y):
            print ("\n\n *****Processing fold ", k_fold_num, " of ", cv.n_splits, " ......")
            X_train_data, X_test_data = X[train_index], X[test_index]
            y_train, y_test = y[train_index], y[test_index]
            X_train = vectorizer.fit_transform(X_train_data)
            X_test = vectorizer.transform(X_test_data)
            if algo == 'logit':
                res = call_logreg_algorithm(X_train, y_train, X_test, y_test)
            elif algo == 'nbsvm':
                res = call_NB_SVM_algorithm(X_train, y_train, X_test, y_test)
            else:
                return 'ERROR'
            results.append(res)
            k_fold_num += 1

#             if k_fold_num ==1:
#                 break
        scores = extract_combined_results(results)
        combined_results[' '.join(pp_step)] = scores
        import pickle
        pickle.dump(combined_results, open("../data/results_logit_1.pkl", "wb"))
    return combined_results, X
#Add the preprocessing step here 
#Add the preprocessing step here
def call_xgboost_to_recursive_transformation_Addition(pp_steps, word_dist_dict_most_common,train_data):
    combined_results = {} #Uncomment 
    for pp_step in pp_steps:
        if len(pp_step) == 1:
            new_mapped_dict = get_corresponding_mapping(word_dist_dict_most_common, pp_step[0])
        else:
            new_mapped_dict = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_step, train_data)
        X = train_data['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
        vocab = set(' '.join(X.values).split())
        vectorizer = TfidfVectorizer(ngram_range=(1,4), min_df=1, vocabulary= vocab)
        results = []
        k_fold_num = 0
        cv = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
        for train_index, test_index in cv.split(X, y):
            print ("\n\n *****Processing fold ", k_fold_num, " of ", cv.n_splits, " ......")
            #print (len(train_index), len(test_index))
            #print("TRAIN:", train_index, "TEST:", test_index)
            X_train_data, X_test_data = X[train_index], X[test_index]
            y_train, y_test = y[train_index], y[test_index]
            X_train = vectorizer.fit_transform(X_train_data)
            X_test = vectorizer.transform(X_test_data)
            res = call_xgboost_algorithm(xgb, vectorizer, X_train, y_train, X_test, y_test)
            results.append(res)
            k_fold_num += 1

    #         if k_fold_num ==1:
    #             break
        scores = extract_combined_results(results)
        combined_results[' '.join(pp_step)] = scores
        import pickle
        pickle.dump(combined_results, open("../data/results_xgboost_1.pkl", "wb"))
    return combined_results, X
def get_words_dict_most_common(df):
    # Get the Word Frequency table
    tmp = df['comment_text'].str.cat(sep=' ')
    words = tmp.split()
    word_dist_dict = nltk.FreqDist(words)
    word_dist_dict_most_common = word_dist_dict.most_common()
    return word_dist_dict_most_common
transformations_orig = [             
            ['convert_to_lower'],
            ['remove_whitespaces'], 
            ['remove_leaky'], 
            ['trim_words_len'],
            ['strip_non_printable_chars'],
            ['replace_abbreviation_words'],
            ['replace_acronyms'],
            ['remove_stopwords'],
            ['remove_rare_words'],
            ['remove_non_alphanumeric'],
            ['remove_non_alphabet_words'],
            ['remove_words_containing_non_alphabets'],
            ['black_listed_words_regex_mapping'],
            ['check_if_proper_name_place_or_ethnicity'],
            ['replace_profane_words_using_fuzzy'],
            ['replace_common_words_using_fuzzy'],
            ['lemmatize_english_words'],
            ['stemming_english_words'],
            ['extract_info_from_url']
                  ]<jupyter_output><empty_output><jupyter_text>Logit<jupyter_code>train_data_copy = train_data.copy()
word_dist_dict_most_common_copy = word_dist_dict_most_common

tr_scores = []
transformations = transformations_orig.copy()
pp_steps = ['black_listed_words_regex_mapping']
max_f1 = 0.7488
tr_scores.append((pp_steps[0], max_f1))
transformations.remove(pp_steps)

new_mapped_dict = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_steps, train_data_copy)
X = train_data_copy['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
train_data_copy['comment_text'] = X
word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)
#Loop through all the transformations to see which one gives the maximum boost to f1_score
while(True):
    tr_iter_score = []
    X_best = None
    tr_best = None
    max_f1_local = max_f1
    for tr in transformations:
        pp_steps = [tr]
        base_res, X = call_logit_to_recursive_transformation_Addition(pp_steps, word_dist_dict_most_common_copy,train_data_copy, 'logit')
        tr_iter_score.append((pp_steps[0][0], base_res))
        if max_f1_local < base_res[tr[0]][0]:
            max_f1_local = base_res[tr[0]][0]
            X_best = X
            tr_best = tr[0]  
            print ('\n\n\n ========= ', tr_best)
    key_neg_f1 = []
    for k in tr_iter_score:
        key_neg_f1.append((k[0], k[1][k[0]][0]))
    ordered = sorted(key_neg_f1,key=itemgetter(1), reverse=True)
    print(ordered)
    if max_f1 >= ordered[0][1]:
        print ('Any other transformation not needed')
        break
    # Add transformation into the    
    tr_scores.append(ordered[0])
    max_f1 = ordered[0][1]
    transformations.remove([ordered[0][0]])
    train_data_copy['comment_text'] = X_best
    word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)
    print(transformations)
    print ('*******************')
    print (tr_scores)
#Logit [('black_listed_words_regex_mapping',  0.7488 )]<jupyter_output><empty_output><jupyter_text>NBSVM<jupyter_code>train_data_copy = train_data.copy()
word_dist_dict_most_common_copy = word_dist_dict_most_common

tr_scores = []
transformations = transformations_orig.copy()
pp_steps = ['replace_common_words_using_fuzzy']
max_f1 = 0.7968
tr_scores.append((pp_steps[0], max_f1))
transformations.remove(pp_steps)

new_mapped_dict = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_steps, train_data_copy)
X = train_data_copy['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
train_data_copy['comment_text'] = X
word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)
#Loop through all the transformations to see which one gives the maximum boost to f1_score
while(True):
    tr_iter_score = []
    X_best = None
    tr_best = None
    max_f1_local = max_f1
    for tr in transformations:
        pp_steps = [tr]
        base_res, X = call_logit_to_recursive_transformation_Addition(pp_steps, word_dist_dict_most_common_copy,train_data_copy, 'nbsvm')
        tr_iter_score.append((pp_steps[0][0], base_res))
        if max_f1_local < base_res[tr[0]][0]:
            max_f1_local = base_res[tr[0]][0]
            X_best = X
            tr_best = tr[0]  
            print ('\n\n\n ========= ', tr_best)
    key_neg_f1 = []
    for k in tr_iter_score:
        key_neg_f1.append((k[0], k[1][k[0]][0]))
    ordered = sorted(key_neg_f1,key=itemgetter(1), reverse=True)
    print(ordered)
    if max_f1 >= ordered[0][1]:
        print ('Any other transformation not needed')
        break
    # Add transformation into the    
    tr_scores.append(ordered[0])
    max_f1 = ordered[0][1]
    transformations.remove([ordered[0][0]])
    train_data_copy['comment_text'] = X_best
    word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)
    print(transformations)
    print ('*******************')
    print (tr_scores)
#NBSVM [('replace_common_words_using_fuzzy',  0.7968 )]<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>train_data_copy = train_data.copy()
word_dist_dict_most_common_copy = word_dist_dict_most_common

tr_scores = []
transformations = transformations_orig.copy()
pp_steps = ['black_listed_words_regex_mapping']
max_f1 = 0.6252
tr_scores.append((pp_steps[0], max_f1))
transformations.remove(pp_steps)

new_mapped_dict = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_steps, train_data_copy)
X = train_data_copy['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
train_data_copy['comment_text'] = X
word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)

#Loop through all the transformations to see which one gives the maximum boost to f1_score
while(True):
    tr_iter_score = []
    for tr in transformations:
        pp_steps = [tr]
        base_res, X = call_xgboost_to_recursive_transformation_Addition(pp_steps, word_dist_dict_most_common_copy,train_data_copy)
        tr_iter_score.append((pp_steps[0][0], base_res))
    key_neg_f1 = []
    for k in tr_iter_score:
        key_neg_f1.append((k[0], k[1][k[0]][0]))
    ordered = sorted(key_neg_f1,key=itemgetter(1), reverse=True)
    print(ordered)
    if max_f1 >= ordered[0][1]:
        print ('Any other transformation not needed')
        break
    # Add transformation into the    
    tr_scores.append(ordered[0])
    max_f1 = ordered[0][1]
    transformations.remove([ordered[0][0]])
    train_data_copy['comment_text'] = X
    word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)
    print(transformations)
    print ('*******************')
    print (tr_scores)
#NBSVM [('black_listed_words_regex_mapping',  0.6252)]<jupyter_output><empty_output><jupyter_text>FastText<jupyter_code>def call_fasttext_to_recursive_transformation_Addition(pp_steps, word_dist_dict_most_common,train_data):
    combined_results = {}
    for pp_step in pp_steps:
        if len(pp_step) == 1:
            new_mapped_dict = get_corresponding_mapping(word_dist_dict_most_common, pp_step[0])
        else:
            new_mapped_dict = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_step, train_data)
        X = train_data['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
        results = []
        k_fold_num = 0
        cv = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
        for train_index, test_index in cv.split(X, y):
            print ("\n\n *****Processing fold ", k_fold_num, " of ", cv.n_splits, " ......")
            X_train_data, X_test_data = X[train_index], X[test_index]
            y_train, y_test = y[train_index], y[test_index]
            texts_train = X_train_data.values
            texts_test  = X_test_data.values
            train = X.copy()
            X_train, X_test, embedding_matrix = preprocess_data_for_fasttext(texts_train, texts_test, train)
            res = call_fasttext_algorithm(X_train, y_train, X_test, y_test, embedding_matrix)
            results.append(res)
            k_fold_num += 1

    #         if k_fold_num ==1:
    #             break
        scores = extract_combined_results(results)
        combined_results[' '.join(pp_step)] = scores
        pickle.dump(combined_results, open("../data/individual_fasttext.pkl", "wb"))
    return combined_results, X
train_data_copy = train_data.copy()
word_dist_dict_most_common_copy = word_dist_dict_most_common

tr_scores = []
transformations = transformations_orig.copy()
pp_steps = ['convert_to_lower']
max_f1 = 0.8055
tr_scores.append((pp_steps[0], max_f1))
transformations.remove(pp_steps)

new_mapped_dict = get_corresponding_mapping_multiple(word_dist_dict_most_common, pp_steps, train_data_copy)
X = train_data_copy['comment_text'].apply(replace_words_from_a_mapping_no_check, args = [new_mapped_dict, 0])
train_data_copy['comment_text'] = X
word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)
#Loop through all the transformations to see which one gives the maximum boost to f1_score
while(True):
    tr_iter_score = []
    for tr in transformations:
        pp_steps = [tr]
        base_res, X = call_fasttext_to_recursive_transformation_Addition(pp_steps, word_dist_dict_most_common_copy,train_data_copy)
        tr_iter_score.append((pp_steps[0][0], base_res))
    key_neg_f1 = []
    for k in tr_iter_score:
        key_neg_f1.append((k[0], k[1][k[0]][0]))
    ordered = sorted(key_neg_f1,key=itemgetter(1), reverse=True)
    print(ordered)
    if max_f1 >= ordered[0][1]:
        print ('Any other transformation not needed')
        break
    # Add transformation into the    
    tr_scores.append(ordered[0])
    max_f1 = ordered[0][1]
    transformations.remove([ordered[0][0]])
    train_data_copy['comment_text'] = X
    word_dist_dict_most_common_copy = get_words_dict_most_common(train_data_copy)
    print(transformations)
    print ('*******************')
    print (tr_scores)<jupyter_output>remove_non_alphabet_words


 *****Processing fold  0  of  10  ......
Epoch 1/3
143613/143613 [==============================] - 179s 1ms/step - loss: 0.1247 - acc: 0.9540
Epoch 2/3
143613/143613 [==============================] - 179s 1ms/step - loss: 0.1024 - acc: 0.9617
Epoch 3/3
143613/143613 [==============================] - 179s 1ms/step - loss: 0.0969 - acc: 0.9632


 *****Processing fold  1  of  10  ......
Epoch 1/3
143613/143613 [==============================] - 179s 1ms/step - loss: 0.1225 - acc: 0.9539
Epoch 2/3
143613/143613 [==============================] - 178s 1ms/step - loss: 0.1012 - acc: 0.9615
Epoch 3/3
143613/143613 [==============================] - 178s 1ms/step - loss: 0.0957 - acc: 0.9631


 *****Processing fold  2  of  10  ......
Epoch 1/3
143613/143613 [==============================] - 179s 1ms/step - loss: 0.1254 - acc: 0.9544
Epoch 2/3
143613/143613 [==============================] - 179s 1ms/step - loss: 0.1011 - acc: 0.9615
Epoch 3/3
143613/143613 [====[...]
{'fraction_non_alphanumeric': 0.12529079428381523, 'fraction_numerical': 0.03169280038671863, 'mean_word_length': 3.9073387694588586, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '113193', 'license': 'MIT', 'n_tokens_mistral': 12280, 'n_tokens_neox': 11482, 'n_words': 1884, 'path': 'src/Preprocessing_paper_Final.ipynb', 'repo_name': 'ifahim/toxic-preprocess', 'size': '33099'}