<jupyter_start><jupyter_text>Read Data  Complete data  Training data<jupyter_code>complete_train_df = pd.read_csv("data/complete_train.csv")
complete_train_df.head(2)<jupyter_output><empty_output><jupyter_text>Test data<jupyter_code>complete_test_df = pd.read_csv("data/complete_test.csv")
complete_test_df.head(2)<jupyter_output><empty_output><jupyter_text>Basic info<jupyter_code>complete_train_df.info()
complete_test_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 5906 entries, 0 to 5905
Data columns (total 4 columns):
 #   Column         Non-Null Count  Dtype 
---  ------         --------------  ----- 
 0   comment_text   5906 non-null   object
 1   identity_hate  5906 non-null   int64 
 2   negative       5906 non-null   int64 
 3   obscene        5906 non-null   int64 
dtypes: int64(3), object(1)
memory usage: 184.7+ KB<jupyter_text>Identity hate sentences<jupyter_code>complete_train_i_df = complete_train_df.loc[complete_train_df.identity_hate==1]
complete_test_i_df = complete_test_df.loc[complete_test_df.identity_hate==1]
print("Training --> Num:{} | Percent:{}".format(len(complete_train_i_df),
                                              round(len(complete_train_i_df)/len(complete_train_df),3)))
print("Test --> Num:{} | Percent:{}".format(len(complete_test_i_df),
                                          round(len(complete_test_i_df)/len(complete_train_df),3)))
complete_train_i_df.head()<jupyter_output>Training --> Num:1144 | Percent:0.023
Test --> Num:137 | Percent:0.003<jupyter_text>Negative sentences<jupyter_code>complete_train_n_df = complete_train_df.loc[complete_train_df.negative==1]
complete_test_n_df = complete_test_df.loc[complete_test_df.negative==1]
print("Training --> Num:{} | Percent:{}".format(len(complete_train_n_df),
                                              round(len(complete_train_n_df)/len(complete_train_df),3)))
print("Test --> Num:{} | Percent:{}".format(len(complete_test_n_df),
                                          round(len(complete_test_n_df)/len(complete_train_df),3)))
complete_train_n_df.head()<jupyter_output>Training --> Num:41504 | Percent:0.851
Test --> Num:4967 | Percent:0.102<jupyter_text>Obscene sentences<jupyter_code>complete_train_o_df = complete_train_df.loc[complete_train_df.obscene==1]
complete_test_o_df = complete_test_df.loc[complete_test_df.obscene==1]
print("Training --> Num:{} | Percent:{}".format(len(complete_train_o_df),
                                              round(len(complete_train_o_df)/len(complete_train_df),3)))
print("Test --> Num:{} | Percent:{}".format(len(complete_test_n_df),
                                          round(len(complete_test_o_df)/len(complete_train_df),3)))
complete_train_o_df.head()<jupyter_output>Training --> Num:6988 | Percent:0.143
Test --> Num:4967 | Percent:0.018<jupyter_text>Subset data  Training data<jupyter_code>train_df = pd.read_csv("data/train.csv")
train_df.head(2)<jupyter_output><empty_output><jupyter_text>Test data<jupyter_code>test_df = pd.read_csv("data/test.csv")
test_df.head(2)<jupyter_output><empty_output><jupyter_text>Basic info<jupyter_code>train_df.info()
test_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 2000 entries, 0 to 1999
Data columns (total 4 columns):
 #   Column         Non-Null Count  Dtype 
---  ------         --------------  ----- 
 0   comment_text   2000 non-null   object
 1   identity_hate  2000 non-null   int64 
 2   negative       2000 non-null   int64 
 3   obscene        2000 non-null   int64 
dtypes: int64(3), object(1)
memory usage: 62.6+ KB<jupyter_text>Identity hate sentences<jupyter_code>train_i_df = train_df.loc[train_df.identity_hate==1]
test_i_df = test_df.loc[test_df.identity_hate==1]
print("Training --> Num:{} | Percent:{}".format(len(train_i_df),
                                              round(len(train_i_df)/len(train_df),3)))
print("Test --> Num:{} | Percent:{}".format(len(test_i_df),
                                          round(len(test_i_df)/len(train_df),3)))
train_i_df.head()<jupyter_output>Training --> Num:1385 | Percent:0.277
Test --> Num:230 | Percent:0.046<jupyter_text>Negative sentences<jupyter_code>train_n_df = train_df.loc[train_df.negative==1]
test_n_df = test_df.loc[test_df.negative==1]
print("Training --> Num:{} | Percent:{}".format(len(train_n_df),
                                              round(len(train_n_df)/len(train_df),3)))
print("Test --> Num:{} | Percent:{}".format(len(test_n_df),
                                          round(len(test_n_df)/len(train_df),3)))
train_n_df.head()<jupyter_output>Training --> Num:1928 | Percent:0.386
Test --> Num:968 | Percent:0.194<jupyter_text>Obscene sentences<jupyter_code>train_o_df = train_df.loc[train_df.obscene==1]
test_o_df = test_df.loc[test_df.obscene==1]
print("Training --> Num:{} | Percent:{}".format(len(train_o_df),
                                              round(len(train_o_df)/len(train_df),3)))
print("Test --> Num:{} | Percent:{}".format(len(test_n_df),
                                          round(len(test_o_df)/len(train_df),3)))
train_o_df.head()<jupyter_output>Training --> Num:2769 | Percent:0.554
Test --> Num:968 | Percent:0.198<jupyter_text>EDA on Subset training data<jupyter_code>train_df = train_df.reset_index().drop(['index'], axis=1)
train_df.head()<jupyter_output><empty_output><jupyter_text>Length of sentences (word counts)<jupyter_code>train_df.comment_text.apply(lambda x:len(x.split())).describe()
test_df.comment_text.apply(lambda x:len(x.split())).describe()<jupyter_output><empty_output><jupyter_text>Min and Max sentence length<jupyter_code>def find_min_max_len(row):
    l = len(row.split())
    if l <= 1 or l >= 1247:
        print(row)
        print("-"*100, "\n")
    return l
train_df.comment_text.apply(find_min_max_len)<jupyter_output>Err:509
---------------------------------------------------------------------------------------------------- 

FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW FAT JEW F[...]<jupyter_text>Cleaning Subset train data  Stopwords<jupyter_code>nltk.download('stopwords')
stopwords = list(nltk.corpus.stopwords.words('english'))
print(sorted(stopwords))<jupyter_output>['a', 'about', 'above', 'after', 'again', 'against', 'ain', 'all', 'am', 'an', 'and', 'any', 'are', 'aren', "aren't", 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'can', 'couldn', "couldn't", 'd', 'did', 'didn', "didn't", 'do', 'does', 'doesn', "doesn't", 'doing', 'don', "don't", 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'hadn', "hadn't", 'has', 'hasn', "hasn't", 'have', 'haven', "haven't", 'having', 'he', 'her', 'here', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'isn', "isn't", 'it', "it's", 'its', 'itself', 'just', 'll', 'm', 'ma', 'me', 'mightn', "mightn't", 'more', 'most', 'mustn', "mustn't", 'my', 'myself', 'needn', "needn't", 'no', 'nor', 'not', 'now', 'o', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 're', 's', 'same', 'shan', "shan't", 'she', "she's", 'should', "should've", 'shouldn', "shouldn't", 'so', 'some',[...]<jupyter_text>Cleaning text  clean object<jupyter_code>clean = CleanText(contractions, stopwords)<jupyter_output><empty_output><jupyter_text>Data before cleaning<jupyter_code>train_df.head()<jupyter_output><empty_output><jupyter_text>Data after cleaning<jupyter_code>train_df["comment_text"] = train_df.comment_text.apply(clean.get_alphabetic)
train_df["comment_text"] = train_df.comment_text.apply(clean.expand_contracted)
train_df["comment_text"] = train_df.comment_text.apply(clean.remove_stopwords)
train_df["comment_text"] = train_df.comment_text.apply(clean.remove_apostrophe)
train_df.head()<jupyter_output><empty_output><jupyter_text>Word Frequency<jupyter_code>def find_word_frequency(row):
    global word_frequency
    word_frequency+= Counter(row.lower().split())
word_frequency = Counter({})
train_df.comment_text.apply(find_word_frequency)
word_frequency = dict(word_frequency)
print(len(word_frequency))
word_frequency = dict(sorted(word_frequency.items(), key=lambda x: x[1], reverse=True))
word_frequency # To see if we can remove any more unimportant most frequent words
sorted(word_frequency.items(), key=lambda x: x[1], reverse=False)<jupyter_output><empty_output><jupyter_text>Read Glove vector<jupyter_code># Read glove vector file
def read_glove_vecs(glove_file):
    with open(glove_file, 'r') as f:
        words = set()
        word_to_vec_map = {}
        for line in f:
            line = line.strip().split()
            curr_word = line[0]
            words.add(curr_word)
            word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64)
        
        i = 1
        words_to_index = {}
        index_to_words = {}
        for w in sorted(words):
            words_to_index[w] = i
            index_to_words[i] = w
            i = i + 1
    return words_to_index, index_to_words, word_to_vec_map
words_to_index, index_to_words, word_to_vec_map = read_glove_vecs('data/glove.6B.50d.txt')
word_to_vec_map["sherrod"]<jupyter_output><empty_output><jupyter_text>Words not present in the embeddings vector(vocab)  Vocab<jupyter_code>vocab = list(words_to_index.keys())<jupyter_output><empty_output><jupyter_text>Words in the training data<jupyter_code>word_list = list(word_frequency.keys())<jupyter_output><empty_output><jupyter_text>Words not in the vocab<jupyter_code>def not_in_vocab(word):
    
    if word not in vocab:
        return (word, word_frequency[word])
non_vocab = list(map(not_in_vocab, word_list))
non_vocab = list(filter(lambda a: a != None, non_vocab))
len(non_vocab)
non_vocab<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1582964289079431, 'fraction_numerical': 0.021765758974842175, 'mean_word_length': 4.0687679083094554, 'pattern_counts': {'":': 0, '<': 90, '<?xml version=': 0, '>': 114, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 49, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '533861', 'license': 'MIT', 'n_tokens_mistral': 4227, 'n_tokens_neox': 3782, 'n_words': 747, 'path': '.ipynb_checkpoints/Sentiment_Preprocessing-checkpoint.ipynb', 'repo_name': 'mohdsaqibhbi/Sentiment_analysis', 'size': '10613'}