<jupyter_start><jupyter_text>**Kfold**<jupyter_code>from sklearn.model_selection import train_test_split

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(X)
x_temp = scaler.transform(X)



# X_train, X_test, y_train, y_test = train_test_split(x_temp, y, test_size = 0.2, random_state = 0)

from sklearn.model_selection import KFold
kfolds = KFold(10, False).split(x_temp)
for _, (X_index, y_index) in enumerate(kfolds):
    '''
      Create tensors for our train / valid / test sets. 
      Need variable to accumulate gradients. 
      First we create tensor, then we will create variable 
    '''
#     model = ANN(input_dim = 24, output_dim = 9, act='relu')
    X_train = X[X_index]
    y_train = y[X_index]
    X_valid = X[y_index]
    y_valid = y[y_index]
    X_train = torch.from_numpy(X_train)
    y_train = torch.from_numpy(y_train)

    # Numpy to Tensor Conversion (Test Set)
    X_valid = torch.from_numpy(X_valid)
    y_valid = torch.from_numpy(y_valid)
    # Make torch datasets from train and test sets
    train = torch.utils.data.TensorDataset(X_train,y_train)
    valid = torch.utils.data.TensorDataset(X_valid,y_valid)

    # Create train and test data loaders
    train_loader = torch.utils.data.DataLoader(train, batch_size = 64, shuffle = True)
    valid_loader = torch.utils.data.DataLoader(valid, batch_size = 64, shuffle = True)
    # Define num epochs
    epochs = 101

    # initialize tracker for minimum validation loss
    valid_loss_min = np.Inf # set initial "min" to infinity

    # Some lists to keep track of loss and accuracy during each epoch
    epoch_list = []
    train_loss_list = []
    val_loss_list = []
    train_acc_list = []
    val_acc_list = []

    # Start epochs
    for epoch in range(epochs):
        # monitor training loss
        train_loss = 0.0
        val_loss = 0.0

        ###################
        # train the model #
        ###################
        # Set the training mode ON -> Activate Dropout Layers
        model.train() # prepare model for training
        # Calculate Accuracy         
        correct = 0
        total = 0

        # Load Train Tuples with Labels(Targets)
        for data, target in train_loader:

            # Convert our feature and labels to Variables to accumulate Gradients
            data = Variable(data).float()
            target = Variable(target).type(torch.LongTensor)

            # clear the gradients of all optimized variables
            optimizer.zero_grad()
            # forward pass: compute predicted outputs by passing inputs to the model
            output = model(data)

            # Calculate Training Accuracy 
            predicted = torch.max(output.data, 1)[1]        
            # Total number of labels
            total += len(target)
            # Total correct predictions
            correct += (predicted == target).sum()

            # calculate the loss
            loss = loss_fn(output, target)
            # backward pass: compute gradient of the loss with respect to model parameters
            loss.backward()
            # perform a single optimization step (parameter update)
            optimizer.step()
            # update running training loss
            train_loss += loss.item()*data.size(0)

        # calculate average training loss over an epoch
        train_loss = np.mean(train_loss)

        # Avg Accuracy
        train_accuracy = 100 * correct / float(total)

        # Put them in their list
        train_acc_list.append(train_accuracy)
        train_loss_list.append(train_loss)


        # Implement Validation like K-fold Cross-validation 
        # Set Evaluation Mode ON -> Turn Off Dropout
        model.eval() # Required for Evaluation/Test

        # Calculate Test/Validation Accuracy         
        correct = 0
        total = 0
        with torch.no_grad():
            for data, target in valid_loader:

                # Convert our features and labels to Variables to accumulate Gradients
                data = Variable(data).float()
                target = Variable(target).type(torch.LongTensor)

                # Predict Output
                output = model(data)

                # Calculate Loss
                loss = loss_fn(output, target)
                val_loss += loss.item()*data.size(0)
                # Get predictions from the maximum value
                predicted = torch.max(output.data, 1)[1]

                # Total number of labels
                total += len(target)

                # Total correct predictions
                correct += (predicted == target).sum()

        # calculate average training loss and accuracy over an epoch
        val_loss = np.mean(val_loss)
        val_accuracy = 100 * correct/ float(total)

        # Put them in their list
        val_acc_list.append(val_accuracy)
        val_loss_list.append(val_loss)
        if epoch % 50 == 0:
            # Print the Epoch and Training Loss Details with Validation Accuracy   
            print('Epoch: {} \tTraining Loss: {:.4f}\t Val. loss: {:.4f} \n Train Acc: {:.2f}\t Val Acc: {:.2f}'.format(
                epoch+1, 
                train_loss,
                val_loss,
                train_accuracy,
                val_accuracy
                ))
            # save model if validation loss has decreased
            if val_loss <= valid_loss_min:
#                 print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(
#                 valid_loss_min,
#                 val_loss))
                torch.save(model.state_dict(), 'model.pt')
                valid_loss_min = val_loss
            # Move to next epoch
            epoch_list.append(epoch + 1)
test_loss = 0
accuracy_test = 0
test_total = 0
correct = 0
for data, target in test_loader:
  # Convert our features and labels to Variables to accumulate Gradients
  data = Variable(data).float()
  target = Variable(target).type(torch.LongTensor)

  # Predict Output
  output = model(data)

  # Calculate Loss
  loss = loss_fn(output, target)
  test_loss += loss.item()*data.size(0)
  # Get predictions from the maximum value
  predicted = torch.max(output.data, 1)[1]

  # Total number of labels
  test_total += len(target)

  # Total correct predictions
  correct += (predicted == target).sum()
 
test_loss = np.mean(test_loss)
accuracy_test = 100 * correct/ float(test_total)
print('Test loss', test_loss)
print('Test Accuracy', accuracy_test)
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
clf.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>RANDOM FOREST CLASSIFIER<jupyter_code>from sklearn.metrics import accuracy_score
predict = clf.predict(X_test)
accuracy_score(y_test, predict)
importances = clf.feature_importances_
import matplotlib.pyplot as plt
%matplotlib inline
std = np.std([tree.feature_importances_ for tree in clf.estimators_],
             axis=0)
indices = np.argsort(importances)

(pd.Series(clf.feature_importances_, index=X_df.columns)
   .nlargest(24)
   .plot(kind='barh'))<jupyter_output><empty_output><jupyter_text>SUPPORT VECTOR MACHINE<jupyter_code>from sklearn.svm import SVC
clf = SVC(gamma='auto')
clf.fit(x_temp, y)
predict = clf.predict(x_temp)
accuracy_score(y, predict)<jupyter_output><empty_output><jupyter_text>XGBOOST CLASSIFIER<jupyter_code>from xgboost import XGBClassifier
model = XGBClassifier(boosting_type='gbdt')
model.fit(X_train, y_train)
predict = model.predict(X_test)
accuracy_score(y_test, predict)
confusion_matrix(y_test, predict)<jupyter_output><empty_output><jupyter_text>LITE GBM<jupyter_code>import lightgbm as lgb
model = lgb.LGBMClassifier(boosting_type='gbdt')
model.fit(X_train, y_train)
predict = model.predict(X_test)
accuracy_score(y_test, predict)
confusion_matrix(y_test, predict)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09344137756394023, 'fraction_numerical': 0.008103317295517852, 'mean_word_length': 2.6417704011065006, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '282218', 'license': 'MIT', 'n_tokens_mistral': 2466, 'n_tokens_neox': 2322, 'n_words': 616, 'path': 'custom_arch.ipynb', 'repo_name': 'nilavghosh/MachineLearning', 'size': '7898'}