<jupyter_start><jupyter_text>local Modeling start with ```python'Tokens_vectors_text/Data_for_modeling.parquet'```> this data has been processed in azure and contains the data and feature vectors ready to model<jupyter_code>pwd
import pyspark.ml.feature 
from pyspark.ml.feature import CountVectorizer
import pyspark.sql.functions  as psf
from pyspark.sql import Window 
from pyspark.sql.types  import DateType
import pandas as pd
#pd.set_option('display.max_colwidth', -1)
from pyspark.sql.types import Row, StructField, StructType, StringType, IntegerType , DateType, ArrayType
#sc.stop()
import findspark
findspark.init()
from pyspark.sql import SQLContext
from pyspark.sql.types import Row, StructField, StructType, StringType, IntegerType , DateType, ArrayType
from pyspark.sql import SparkSession
from pyspark import SparkContext ,SparkConf
SparkContext.setSystemProperty('spark.executor.memory', '6g')
#SparkContext.setSystemProperty('spark.rdd.compress', True)
SparkContext.setSystemProperty('spark.driver.memory', '16g')
sc = SparkContext(master="local[4]")
print(sc)

from pyspark.ml import Pipeline
from pyspark.ml.classification import LogisticRegression, LinearSVC,NaiveBayes
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.feature import HashingTF, Tokenizer
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.classification import RandomForestClassifier
train_path='/home/ucsddse230/work/final_project/post_azure/training_features.parquet'
test_path='/home/ucsddse230/work/final_project/post_azure/testing_features.parquet'
model_path='/home/ucsddse230/work/final_project/model'
dl_path='/home/ucsddse230/work/final_project/dl_model/'
spark = SparkSession.builder.getOrCreate()
train_data= spark.read.parquet(train_path)
test_data= spark.read.parquet(test_path)<jupyter_output><empty_output><jupyter_text>happy modeling>please explore the the multilayer perceptron here and many othershttps://spark.apache.org/docs/2.3.0/ml-classification-regression.html<jupyter_code>train_data.columns
print(train_data.count(),len(train_data.columns))
print(test_data.count(),len(test_data.columns))
pos_count = train_data[train_data.label==1].count()
neg_count = train_data[train_data.label==0].count()
train_data.head()
from utils import train, evaluate, display_metrics, classification_metrics
#Gradient Boosted Trees
from pyspark.ml.classification import GBTClassifier
gbt = GBTClassifier(maxIter=100)
gbtModel = gbt.fit(train_data)
print(gbt.explainParams())
#Train prediction
predictions =gbtModel.transform(train_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("GBT",Y_pred,Y_true)
#Test prediction
predictions =gbtModel.transform(test_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("GBT",Y_pred,Y_true)
gbtModel.save(model_path)
#Logistic Regression
from pyspark.ml.classification import LogisticRegression
lr = LogisticRegression(maxIter=100)
lrModel = lr.fit(train_data)
print(lr.explainParams())
#Train prediction
predictions =lrModel.transform(train_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("Logistic Regression",Y_pred,Y_true)
#Test prediction
predictions =lrModel.transform(test_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("Logistic Regression",Y_pred,Y_true)
#SVC
from pyspark.ml.classification import LinearSVC
lsvc = LinearSVC(maxIter=100)
svcModel = lsvc.fit(train_data)
print(lsvc.explainParams())
#Train prediction
predictions = svcModel.transform(train_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("SVM",Y_pred,Y_true)
#Test prediction
predictions = svcModel.transform(test_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("SVM",Y_pred,Y_true)
from pyspark.ml.classification import NaiveBayes
nb = NaiveBayes(smoothing=1.0, modelType="multinomial",thresholds=[neg_count,pos_count])

# train the model
nbModel = nb.fit(train_data)
print(nb.explainParams())
#Train metrics
predictions = nbModel.transform(train_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("Naive Bayes",Y_pred,Y_true)
#Test prediction
predictions = nbModel.transform(test_data)
Y_true=np.array(predictions.select('label').collect())
Y_pred=np.array(predictions.select('prediction').collect())
display_metrics("Naive Bayes",Y_pred,Y_true)
#Uncomment below line to install torch
#!pip install torch
#DL libraries and pandas and numpy
import numpy as np
import pandas as pd
from scipy import sparse
import torch
from torch.utils.data import TensorDataset, Dataset

#Convert to Pandas dataframe
train_df = train_data.toPandas()
test_df = test_data.toPandas()
print(train_df.shape,test_df.shape)
train_y = np.array(train_df['label'].values)
test_y = np.array(test_df['label'].values)
#Create tensor datasets for MLP
#train dataset
target = torch.from_numpy(train_y.astype('long')).view(-1)

series = train_df['features'].apply(lambda x : np.array(x.toArray())).as_matrix().reshape(-1,1)
features = np.apply_along_axis(lambda x : x[0], 1, series)
train_dataset = TensorDataset(torch.from_numpy(features.astype('float32')), target)

#test dataset
target = torch.from_numpy(test_y.astype('long')).view(-1)

series = test_df['features'].apply(lambda x : np.array(x.toArray())).as_matrix().reshape(-1,1)
features = np.apply_along_axis(lambda x : x[0], 1, series)
test_dataset = TensorDataset(torch.from_numpy(features.astype('float32')), target)

train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=0)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=True, num_workers=0)
import torch.nn as nn
import torch.nn.functional as F

class MyMLP(nn.Module):
    def __init__(self):
        super(MyMLP, self).__init__()
        self.hidden1 = nn.Linear(4499,2056)
        self.hidden2 = nn.Linear(2056,1028)
        self.hidden3 = nn.Linear(1028,512)
        self.hidden4 = nn.Linear(512,256)
        self.hidden5 = nn.Linear(256,64)
        self.hidden6 = nn.Linear(64,16)
        self.out = nn.Linear(16,2)

    def forward(self, x):
        x = F.relu(self.hidden1(x))
        x = F.relu(self.hidden2(x))
        x = F.relu(self.hidden3(x))
        x = F.relu(self.hidden4(x))
        x = F.relu(self.hidden5(x))
        x = F.relu(self.hidden6(x))
        x = self.out(x)
        return x
import torch.optim as optim
model = MyMLP()
save_file = 'MyMLP.pth'
class_weightage = torch.tensor([pos_count/neg_count,1-pos_count/neg_count])
print(class_weightage)
criterion = nn.CrossEntropyLoss(class_weightage)
optimizer = optim.Adam(model.parameters())

best_val_acc = 0.0
NUM_EPOCHS=3
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
best_auc = 0.0
train_losses, train_accuracies = [], []
test_losses, test_accuracies = [], []
for epoch in range(NUM_EPOCHS):
    train_loss, train_accuracy = train(model, device, train_loader, criterion, optimizer, epoch)
    test_loss, test_accuracy, test_results = evaluate(model, device, test_loader, criterion, print_freq=1000)

    train_losses.append(train_loss)
    test_losses.append(test_loss)

    train_accuracies.append(train_accuracy)
    test_accuracies.append(test_accuracy)
    
    Y_true=np.array(test_results)[:,0]
    Y_pred=np.array(test_results)[:,1]
    acc, auc_, precision, recall, f1score = classification_metrics(Y_pred,Y_true)

    is_best = auc_ > best_auc  # let's keep the model that has the best AUC
    if is_best:
        print(auc_,best_auc)
        best_auc = auc_
        torch.save(model, (dl_path + save_file))

#Load model
best_model = torch.load(dl_path + save_file)
#Train metrics
train_loss, train_accuracy, train_results = evaluate(best_model, device, train_loader, criterion, print_freq=1000)
Y_true=np.array(train_results)[:,0]
Y_pred=np.array(train_results)[:,1]
display_metrics("MLP",Y_pred,Y_true)
#Test metrics
test_loss, test_accuracy, test_results = evaluate(best_model, device, test_loader, criterion, print_freq=900000)
Y_true=np.array(test_results)[:,0]
Y_pred=np.array(test_results)[:,1]
display_metrics("MLP",Y_pred,Y_true)
#Create tensor datasets for CNN
#train dataset
target = torch.from_numpy(train_y.astype('long'))

series = train_df['features'].apply(lambda x : np.array(x.toArray())).as_matrix().reshape(-1,1)
features = np.apply_along_axis(lambda x : x[0], 1, series)
train_dataset = TensorDataset(torch.from_numpy(features.astype('float32')).unsqueeze(1), target)

#test dataset
target = torch.from_numpy(test_y.astype('long'))

series = test_df['features'].apply(lambda x : np.array(x.toArray())).as_matrix().reshape(-1,1)
features = np.apply_along_axis(lambda x : x[0], 1, series)
test_dataset = TensorDataset(torch.from_numpy(features.astype('float32')).unsqueeze(1), target)

train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=0)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=True, num_workers=0)
class MyCNN(nn.Module):
    def __init__(self):
        super(MyCNN, self).__init__()
        self.conv1 = nn.Conv1d(in_channels=1, out_channels=6, kernel_size=5)
        self.pool = nn.MaxPool1d(kernel_size=2,stride=2)
        self.conv2 = nn.Conv1d(in_channels=6, out_channels=16, kernel_size=5)
        self.fc1 = nn.Linear(in_features=1121, out_features=64)
        self.fc2 = nn.Linear(64, 32)
        self.fc3 = nn.Linear(32, 2)
#m1: [128 x 4484], m2: [128 x 64]
    def forward(self, x):
#torch.Size([32, 16, 1121])
#torch.Size([32, 1121])
#torch.Size([32, 17936])

        x = self.pool(torch.nn.functional.relu(self.conv1(x)))        
        x = self.pool(torch.nn.functional.relu(self.conv2(x)))
        
        #flatten
        x = x[:,0]        
        x = F.relu(self.fc1(x))        
        x = F.dropout(x, training=self.training)        
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        
        return x
model = MyCNN()
save_file = 'MyCNN.pth'
class_weightage = torch.tensor([pos_count/neg_count,1-pos_count/neg_count])
print(class_weightage)
criterion = nn.CrossEntropyLoss(class_weightage)
optimizer = optim.Adam(model.parameters())

best_val_acc = 0.0
NUM_EPOCHS=3
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
best_auc = 0.0
train_losses, train_accuracies = [], []
test_losses, test_accuracies = [], []
for epoch in range(NUM_EPOCHS):
    train_loss, train_accuracy = train(model, device, train_loader, criterion, optimizer, epoch)
    test_loss, test_accuracy, test_results = evaluate(model, device, test_loader, criterion, print_freq=1000)

    train_losses.append(train_loss)
    test_losses.append(test_loss)

    train_accuracies.append(train_accuracy)
    test_accuracies.append(test_accuracy)
    
    Y_true=np.array(test_results)[:,0]
    Y_pred=np.array(test_results)[:,1]
    acc, auc_, precision, recall, f1score = classification_metrics(Y_pred,Y_true)

    is_best = auc_ > best_auc  # let's keep the model that has the best AUC
    if is_best:
        print(auc_,best_auc)
        best_auc = auc_
        torch.save(model, (dl_path + save_file))

#Load model
best_model = torch.load(dl_path + save_file)
#Train metrics
train_loss, train_accuracy, train_results = evaluate(best_model, device, train_loader, criterion, print_freq=1000)
Y_true=np.array(train_results)[:,0]
Y_pred=np.array(train_results)[:,1]
display_metrics("CNN",Y_pred,Y_true)
#Test metrics
test_loss, test_accuracy, test_results = evaluate(best_model, device, test_loader, criterion, print_freq=900000)
Y_true=np.array(test_results)[:,0]
Y_pred=np.array(test_results)[:,1]
display_metrics("CNN",Y_pred,Y_true)<jupyter_output>______________________________________________
Classifier: CNN
Accuracy: 0.721554564838147
AUC: 0.6051069203872325
Precision: 0.10715544589354953
Recall: 0.4720496894409938
F1-score: 0.17466245331801206
______________________________________________
{'fraction_non_alphanumeric': 0.12758840395030263, 'fraction_numerical': 0.0269194010831475, 'mean_word_length': 6.138715179079022, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '327908', 'license': 'MIT', 'n_tokens_mistral': 4584, 'n_tokens_neox': 4364, 'n_words': 760, 'path': 'post_azure/local_modeling_with_azure_features.ipynb', 'repo_name': 'ghoersti/Natural-Language-Processing-for-predicting-hospital-readmission', 'size': '12556'}