<jupyter_start><jupyter_text>Data Reading<jupyter_code>import pandas as pd
pd.set_option('display.max_colwidth',100)
Data=pd.read_excel('drive/My Drive/Summer Internship 2020 July/My Test File/Ankit/2020 Dataset/new data/hasoc_2020_hi_train.xlsx',sep='\t')
Data.head()<jupyter_output><empty_output><jupyter_text>Data PreprocessingStop Word Removal<jupyter_code>#rem0ving emoji

import re

def deEmojify(txt):
    regrex_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           "]+", flags = re.UNICODE)
    return regrex_pattern.sub(r'',txt)

Data['text']=Data['text'].apply(lambda x:deEmojify(x))
Data[['text','task1']].head()<jupyter_output><empty_output><jupyter_text>Removing URL<jupyter_code># remove URL
def remove_URL(txt):
    url= re.compile(r"https?://\S+|www\.\S+")
    return url.sub(r"",txt)

Data['text']=Data['text'].apply(lambda x:remove_URL(x))
Data[['text']].head()

Data['text']=Data['text'].str.replace(r'\d+','')
Data[['text','task1']].head()<jupyter_output><empty_output><jupyter_text>Tokenization, Encoding, padding after preprocessing<jupyter_code>import keras
from keras.preprocessing.text import Tokenizer
tok1 = Tokenizer(char_level=False, filters ='!"$%&@()*+,-./:;”“<=>?[\\]^_`{|}~\t\n', lower = True)
tok1.fit_on_texts(Data['text'])
words=len(tok1.word_counts)
words
word_list=[tok1.word_index]
#word_list
encoded=tok1.texts_to_sequences(Data['text'])
#print(Data['text'][0])
#encoded[1211]
##         Padding encoded sequence of words
from keras.preprocessing import sequence
max_length=20
padd = sequence.pad_sequences(encoded, maxlen=max_length, padding='post')
#padd[0:12]<jupyter_output><empty_output><jupyter_text>Label Encoding<jupyter_code>import numpy as np
from keras.utils.np_utils import to_categorical
from sklearn import preprocessing
labelEncode=preprocessing.LabelEncoder()
labelEncode.fit(Data['task1'])
print (labelEncode.classes_)
train_labelEncode=labelEncode.transform(Data['task1'])
train_label=to_categorical(np.asarray(train_labelEncode))<jupyter_output>['HOF' 'NOT']<jupyter_text>Importing libraries for Model creation<jupyter_code># importing libraries for creating model
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Embedding,MaxPool1D
from keras.layers import Conv1D, GlobalMaxPooling1D, MaxPooling1D<jupyter_output><empty_output><jupyter_text>CNN model<jupyter_code>model = Sequential()
model.add(Embedding(input_dim = words + 1, output_dim = 100,input_length=max_length))
model.add(Conv1D(filters =512,kernel_size=2, activation='relu'))
model.add(MaxPooling1D(3))
model.add(Flatten())
model.add(Dense(128, activation = 'relu'))
model.add(Dense(2, activation = 'softmax'))
model.summary()<jupyter_output>Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_1 (Embedding)      (None, 20, 100)           1354600   
_________________________________________________________________
conv1d_1 (Conv1D)            (None, 19, 512)           102912    
_________________________________________________________________
max_pooling1d_1 (MaxPooling1 (None, 6, 512)            0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 3072)              0         
_________________________________________________________________
dense_2 (Dense)              (None, 128)               393344    
_________________________________________________________________
dense_3 (Dense)              (None, 2)                 258       
======================================================[...]<jupyter_text>Model Compilation<jupyter_code>## compile

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])<jupyter_output><empty_output><jupyter_text>Model Training<jupyter_code>model.fit(padd,train_label,epochs=15,verbose=1,batch_size= 32)<jupyter_output>Epoch 1/15
93/93 [==============================] - 1s 14ms/step - loss: 0.5984 - acc: 0.7067
Epoch 2/15
93/93 [==============================] - 1s 14ms/step - loss: 0.3621 - acc: 0.8296
Epoch 3/15
93/93 [==============================] - 1s 14ms/step - loss: 0.0620 - acc: 0.9862
Epoch 4/15
93/93 [==============================] - 1s 14ms/step - loss: 0.0283 - acc: 0.9960
Epoch 5/15
93/93 [==============================] - 1s 13ms/step - loss: 0.0113 - acc: 0.9980
Epoch 6/15
93/93 [==============================] - 1s 14ms/step - loss: 0.0087 - acc: 0.9983
Epoch 7/15
93/93 [==============================] - 1s 13ms/step - loss: 0.0073 - acc: 0.9983
Epoch 8/15
93/93 [==============================] - 1s 13ms/step - loss: 0.0075 - acc: 0.9983
Epoch 9/15
93/93 [==============================] - 1s 13ms/step - loss: 0.0063 - acc: 0.9983
Epoch 10/15
93/93 [==============================] - 1s 13ms/step - loss: 0.0058 - acc: 0.9987
Epoch 11/15
93/93 [==============================] - 1s 13m[...]<jupyter_text>Test data reading<jupyter_code>Test =pd.read_csv('drive/My Drive/Summer Internship 2020 July/My Test File/Ankit/2020 Dataset/test data/hindi_test_1509.csv')
Test.dropna(inplace = True)
Test.head()<jupyter_output><empty_output><jupyter_text>Test data preprocessing<jupyter_code>## doing preprocessing as above
Test['text']=Test['text'].apply(lambda x:deEmojify(x))
Test['text']=Test['text'].apply(lambda x: remove_URL(x))
Test['text']=Test['text'].str.replace(r'\d+','')
t= Test['text']<jupyter_output><empty_output><jupyter_text>Test data encoding, padding<jupyter_code>encoded1 =tok1.texts_to_sequences(t)
padded = sequence.pad_sequences(encoded1, maxlen=max_length, padding='post')
padded[0]<jupyter_output><empty_output><jupyter_text>Test label encoding<jupyter_code>test_labelEncode=labelEncode.transform(Test['task1'])
test_label=to_categorical(np.asarray(test_labelEncode))<jupyter_output><empty_output><jupyter_text>Test data Model prediction<jupyter_code>test_predictions=model.predict(padded)
test_predictions2=np.argmax(test_predictions,axis=1)<jupyter_output><empty_output><jupyter_text>Test data Classification report<jupyter_code>#Accuracy 
from sklearn.metrics import classification_report
print(classification_report(test_labelEncode,test_predictions2))<jupyter_output>precision    recall  f1-score   support

           0       0.62      0.41      0.50       197
           1       0.78      0.89      0.83       466

    accuracy                           0.75       663
   macro avg       0.70      0.65      0.67       663
weighted avg       0.74      0.75      0.73       663<jupyter_text>Adding predicted lable as a colum to dataFrame 'Test'<jupyter_code>Test['Predictionlabel']=labelEncode.inverse_transform(test_predictions2)
Test.head()<jupyter_output><empty_output><jupyter_text>Generating CSV file<jupyter_code>HindiTask1File=Test[['tweet_id','Predictionlabel','ID']]
HindiTask1File.head()
HindiTask1File.to_csv('submission_HI_A.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18722345132743362, 'fraction_numerical': 0.06319137168141593, 'mean_word_length': 5.458035714285714, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '387645', 'license': 'Apache-2.0', 'n_tokens_mistral': 2736, 'n_tokens_neox': 2411, 'n_words': 396, 'path': 'BEST MODELS/Hindi sub-task A.ipynb', 'repo_name': 'ankitmishra2232/HASOC-2020-competition', 'size': '7232'}