<jupyter_start><jupyter_text><jupyter_code>%reset
# # Load the Drive helper and mount
from google.colab import drive

# # This will prompt for authorization.
drive.mount('/content/drive')
import keras
import numpy as np
import random as rn
rn.seed(123)
np.random.seed(123)
from keras.models import Model
from keras.layers import Input, Dense,Conv2D ,LSTM,Lambda,Flatten,Dropout
from keras.utils import np_utils
from keras import backend as K
tf_session = K.get_session()
from keras import regularizers
import tensorflow as tf
tf.set_random_seed(123)
def stack_dim(X):
    conv_outputs=tf.unstack(X,axis=3)
    return tf.concat(conv_outputs,axis=1)

def get_shape(input_shape):
    assert len(input_shape)==4
    return (input_shape[0],input_shape[1]*input_shape[3],input_shape[2])

def model(input_shape,hidden_size,dense_layer1,dense_layer2):
    X_input=Input(input_shape)
    X=Conv2D(32, (5, 5), input_shape=input_shape, activation='relu')(X_input)
    X=Dropout(0.5)(X)
    X=Lambda(stack_dim,output_shape=get_shape)(X)
    X=LSTM(hidden_size,return_sequences=True)(X)
    X=Dense(dense_layer1, activation='relu',kernel_regularizer=regularizers.l2(0.01))(X)
    X=Flatten()(X)
    X=Dropout(0.5)(X)
    X=Dense(dense_layer2, activation='softmax')(X)
    model = Model(inputs = X_input, outputs = X, name='sent_classifier')
    return model
X_train=np.load('drive/My Drive/SC-Project/train_data.npy')
X_test=np.load('drive/My Drive/SC-Project/X_test.npy')
y_train=np.load('drive/My Drive/SC-Project/y_train.npy')
y_test=np.load('drive/My Drive/SC-Project/y_test.npy')
m=model(X_train.shape[1:],128,50,2)
from keras.optimizers import SGD,Adam
# opt=SGD(lr=0.05, momentum=0.01,decay=0.0, nesterov=False)
m.compile(optimizer='sgd',loss='binary_crossentropy',metrics=['accuracy'])
def train(l,h):
    train_size=h-l
    train_pos=np.where(y_train[l:h][...,1]==1)[0]+l
    train_neg=np.where(y_train[l:h][...,0]==1)[0]+l
    np.random.seed(123)
    pos=np.random.choice(train_pos,train_size//2)
    np.random.seed(123)
    neg=np.random.choice(train_neg,train_size-train_size//2)
    indices=np.append(pos,neg,0)
    np.random.seed(123)
    np.random.shuffle(indices)
    return indices
train_index=train(0,2210)
val_index=train(4000,8000)
val_index
from sklearn.utils import resample
ups_train_index = resample(train_index,replace=True,n_samples=4000,random_state=123)
len(ups_train_index)
hist=m.fit(X_train[ups_train_index],y_train[ups_train_index],batch_size=64,epochs=20,verbose=1,validation_data=(X_train[val_index],y_train[val_index]))
m.save('drive/My Drive/SC-Project/final.h5')
from keras.models import load_model
m = load_model('drive/My Drive/SC-Project/final.h5',custom_objects={"tf": tf})
pred=m.evaluate(X_test,y_test)
print(pred)
m.summary()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1503948312993539, 'fraction_numerical': 0.03374012921751615, 'mean_word_length': 6.7202216066482, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '145395', 'license': 'MIT', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1090, 'n_words': 151, 'path': 'model.ipynb', 'repo_name': 'shreyasseshadri/SC-Project', 'size': '2786'}