<jupyter_start><jupyter_text><jupyter_code>numeric_features = csr_matrix(filtered_data[[ 'summary_size', 'words_in_summary', 'Price', 'ex_flag', 'new_flag', 'go_flag']].values)
train_idf = hstack((train_tfidf, numeric_features)).toarray()
y = filtered_data['Condition']
X_train, X_valid, y_train, y_valid = train_test_split(train_idf, y, test_size=0.2, random_state=1, stratify = y )
classifier = RidgeClassifier(alpha=.0001)
classifier.fit(X_train, y_train)
pred_tfidf = classifier.decision_function(X_valid)
probs_tfidf = np.exp(pred_tfidf)
pred_tfidf = np.where(probs_tfidf>0.90, 1, 0)
accuracy_val = accuracy_score(y_valid, pred_tfidf)
accuracy_val
filtered_data['Condition'].value_counts(normalize=True)
gnb = GaussianNB()
y_pred = gnb.fit(X_train, y_train).predict(X_valid)
accuracy_gb = accuracy_score(y_valid, y_pred)
print("Gaussian Naive Bayes Accuracy Score -> ",accuracy_gb*100)
Naive = naive_bayes.MultinomialNB()
Naive.fit(X_train,y_train)
predictions_NB = Naive.predict(X_valid)
print("Naive Bayes Accuracy Score -> ",accuracy_score(predictions_NB, y_valid)*100)
# SVM = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
# SVM.fit(X_train,y_train)
# predictions_SVM = SVM.predict(X_valid)# Use accuracy_score function to get the accuracy
# print("SVM Accuracy Score -> ",accuracy_score(predictions_SVM, y_valid)*100)
#Random Forest Classifier
random_forest = RandomForestClassifier(n_estimators=50)
y_pred_rf = random_forest.fit(X_train,y_train).predict(X_valid)
accuracy_rf = accuracy_score(y_pred_rf, y_valid)
print("Random Forest Accuracy Score -> ",accuracy_rf*100)

import pandas as pd
from keras.preprocessing import text,sequence
import numpy as np
from keras.layers import GRU, Conv1D,CuDNNGRU
from keras.layers import Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D
from keras.callbacks import Callback,EarlyStopping
from sklearn.model_selection import train_test_split
from keras.preprocessing.text import Tokenizer
from keras.callbacks import ModelCheckpoint
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, Embedding, Dropout, Conv1D
from keras.models import Model, load_model
from tensorflow.keras.optimizers import Adam
from sklearn.metrics import accuracy_score,roc_auc_score,f1_score
import re,string
from sklearn.linear_model import LogisticRegression, RidgeClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import FeatureUnion
from scipy import sparse
from scipy.sparse import hstack, csr_matrix
from keras.models import model_from_json
import warnings
warnings.filterwarnings("ignore")
filtered_data.head()
# !wget https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M.vec.zip
# !unzip Text\ classification/crawl-300d-2M.vec.zip
embed_size = 300
max_features = 6500
maxlen = 350
tokenizer = text.Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(list(filtered_data['Description']))
EMBEDDING_FILE = 'Text classification/crawl-300d-2M.vec'
print('loading embedding file')
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE,encoding="utf8"))
print('Done loading embedding file')
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros((nb_words, embed_size))
for word, i in word_index.items():
  if i >= max_features: continue
  embedding_vector = embeddings_index.get(word)
  if embedding_vector is not None: embedding_matrix[i] = embedding_vector
def build_model(body_flag,body_flag_test, X_train,Y_train,X_valid,Y_valid,check_point,early_stop,file_path, lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
    body_text_flag = body_flag
    inp = Input(shape = (maxlen,))
    print(nb_words)
    x = Embedding(max_features , embed_size, weights = [embedding_matrix], trainable = False)(inp)

    x = SpatialDropout1D(dr)(x)

    x = Bidirectional(GRU(64, return_sequences = True))(x)
    x = Conv1D(64, kernel_size = 1, padding = "valid", kernel_initializer = "he_uniform")(x)
    avg_pool = GlobalAveragePooling1D()(x)
    max_pool = GlobalMaxPooling1D()(x)
    
    # body_text_flag = Input(shape=[body_text_flag.shape[1]], name="body_text_flag")
    x = concatenate([avg_pool, max_pool])
    x = Dense(1, activation = "sigmoid")(x)
    model = Model(inputs = inp, outputs = x)
    ra_val = RocAucEvaluation(validation_data=(X_valid, Y_valid), interval = 1)
    model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
    history = model.fit(X_train , Y_train, batch_size = 64, epochs = 15, validation_data = (X_valid, Y_valid), 
                        verbose = 1, callbacks = [ra_val, check_point, early_stop])
    model = load_model(file_path)
    return model

class RocAucEvaluation(Callback):
    def __init__(self, validation_data=(), interval=1):
        super(Callback, self).__init__()

        self.interval = interval
        self.X_val, self.y_val = validation_data

    def on_epoch_end(self, epoch, logs={}):
        if epoch % self.interval == 0:
            y_pred = self.model.predict(self.X_val, verbose=0)
            score = roc_auc_score(self.y_val, y_pred)
            print("\n ROC-AUC - epoch: %d - score: %.6f \n" % (epoch+1, score))
def build_model(body_flag,body_flag_test, X_train,Y_train,X_valid,Y_valid,check_point,early_stop,file_path, lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
    body_text_flag = body_flag
    inp = Input(shape = (maxlen,))
    x = Embedding(max_features, embed_size, weights = [embedding_matrix], trainable = False)(inp)
    x = SpatialDropout1D(dr)(x)

    x = Bidirectional(GRU(units, return_sequences = True))(x)
    x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform")(x)
    avg_pool = GlobalAveragePooling1D()(x)
    max_pool = GlobalMaxPooling1D()(x)
    body_text_flag = Input(shape=[body_text_flag.shape[1]], name="body_text_flag")
    x = concatenate([avg_pool, max_pool, body_text_flag])
    x = Dense(1, activation = "sigmoid")(x)
    model = Model(inputs = [inp, body_text_flag], outputs = x)
    ra_val = RocAucEvaluation(validation_data=([X_valid,body_flag_test], Y_valid), interval = 1)
    model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
    history = model.fit([X_train,body_flag] , Y_train, batch_size = 64, epochs = 15, validation_data = ([X_valid,body_flag_test], Y_valid), 
                        verbose = 1, callbacks = [ra_val, check_point, early_stop])
    model = load_model(file_path)
    return model

class RocAucEvaluation(Callback):
    def __init__(self, validation_data=(), interval=1):
        super(Callback, self).__init__()

        self.interval = interval
        self.X_val, self.y_val = validation_data

    def on_epoch_end(self, epoch, logs={}):
        if epoch % self.interval == 0:
            y_pred = self.model.predict(self.X_val, verbose=0)
            score = roc_auc_score(self.y_val, y_pred)
            print("\n ROC-AUC - epoch: %d - score: %.6f \n" % (epoch+1, score))
X_train, X_valid, Y_train, Y_valid = train_test_split(filtered_data.drop(['Condition'], axis=1), y, test_size = 0.2, random_state =1,stratify=y)
raw_text_train = 'Title' + X_train["Title"].astype(str) + ' ' + X_train["Description"]
raw_text_valid = 'Title' + X_valid["Title"].astype(str) + ' ' + X_valid["Description"]

tk = text.Tokenizer(num_words = max_features, lower = True)
tk.fit_on_texts(raw_text_train)
X_train["text_seq"] = tk.texts_to_sequences(raw_text_train)
X_valid["text_seq"] = tk.texts_to_sequences(raw_text_valid)

print('padding')
raw_text_train = pad_sequences(X_train.text_seq, maxlen = maxlen)
raw_text_test = pad_sequences(X_valid.text_seq, maxlen = maxlen)


file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_acc", verbose = 1,
                      save_best_only = True, mode = "max")
early_stop = EarlyStopping(monitor = "val_acc", mode = "max", patience = 5)
test_flag = X_train[['ex_flag', 'new_flag', 'go_flag']].values
body_flag_test = X_valid[['ex_flag', 'new_flag', 'go_flag']].values
model = build_model(test_flag,body_flag_test,raw_text_train,Y_train,raw_text_test,Y_valid,check_point,early_stop,file_path,lr = 1e-3, lr_d = 0, units = 64, dr = 0.2)

model.load_weights(file_path)
predictions = model.predict([raw_text_test,body_flag_test]).reshape((raw_text_test.shape[0],))

# pred = loaded_model.predict(X_valid, batch_size = 1024, verbose = 1)
pred = np.where(predictions>=0.5,1,0)
score = pd.DataFrame(pred)
# print(Y_valid,score.astype(int))
accuracy_val = accuracy_score(Y_valid, score.values)
print('NN OOF accuracy: {}'.format(accuracy_val))
inp = Input(shape = (maxlen,))
x = Embedding(nb_words, embed_size, weights = [embedding_matrix], trainable = False)(inp)

x = SpatialDropout1D(dr)(x)

x = Bidirectional(GRU(units, return_sequences = True))(x)
x = Conv1D(64, kernel_size = 1, padding = "valid", kernel_initializer = "he_uniform")(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)

x = concatenate([avg_pool, max_pool])
x = Dense(1, activation = "sigmoid")(x)
model = Model(inputs = [inp, body_text_flag], outputs = x)
ra_val = RocAucEvaluation(validation_data=([X_valid,body_flag_test], Y_valid), interval = 1)
model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
history = model.fit(X_train , Y_train, batch_size = 64, epochs = 15, validation_data = (X_valid, Y_valid), 
                    verbose = 1, callbacks = [ra_val, check_point, early_stop])

model = load_model(file_path)
model.load_weights(file_path)
predictions = model.predict([raw_text_test,body_flag_test]).reshape((raw_text_test.shape[0],))

# pred = loaded_model.predict(X_valid, batch_size = 1024, verbose = 1)
pred = np.where(predictions>=0.5,1,0)
score = pd.DataFrame(pred)
# print(Y_valid,score.astype(int))
accuracy_val = accuracy_score(Y_valid, score.values)
print('NN OOF accuracy: {}'.format(accuracy_val))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12921293111919033, 'fraction_numerical': 0.016114768595853396, 'mean_word_length': 5.687253613666229, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '462969', 'license': 'MIT', 'n_tokens_mistral': 3824, 'n_tokens_neox': 3636, 'n_words': 706, 'path': 'Text Analysis Code/TextClassification_Part2.ipynb', 'repo_name': 'guptaneha92/Craigslist_listing_classifier', 'size': '10177'}