<jupyter_start><jupyter_text>Prepare the dataset Tokanize and all<jupyter_code>## libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
import requests
import re
# mounting drive
from google.colab import drive
drive.mount('/content/drive/')
dataDir = 'drive/MyDrive/git/Datasets/'
## loading
df = pd.read_csv(dataDir+'emails.csv', index_col="Email No.")
df.head()
## checking for ham/spam SMS ratio from the whole bulk
df['Prediction'].value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>Data Cleaning & Preparation<jupyter_code>df.shape
df.drop_duplicates(inplace=True)
df.head()
df.shape
## taken from GeeksForGeek

# are duplicates.
def getDuplicateColumns(df):
  
    # Create an empty set
    duplicateColumnNames = set()
      
    # Iterate through all the columns 
    # of dataframe
    for x in range(df.shape[1]):
          
        # Take column at xth index.
        col = df.iloc[:, x]
          
        # Iterate through all the columns in
        # DataFrame from (x + 1)th index to
        # last index
        for y in range(x + 1, df.shape[1]):
              
            # Take column at yth index.
            otherCol = df.iloc[:, y]
              
            # Check if two columns at x & y
            # index are equal or not,
            # if equal then adding 
            # to the set
            if col.equals(otherCol):
                duplicateColumnNames.add(df.columns.values[y])
                  
    # Return list of unique column names 
    # whose contents are duplicates.
    return list(duplicateColumnNames)
# dupCols = getDuplicateColumns(df)
#word_counts = word_counts.drop_duplicates(columns=dupCols)
df.shape<jupyter_output><empty_output><jupyter_text>---  ---  Data Praparation<jupyter_code>def groupCol(dataCol):
  dataCol = pd.Series(dataCol)
  return [str(val) for val in dataCol//10]
def prepare(df):
  ## grouping all cols
  for col in tuple(df.columns[:-1]):
    df[col] = groupCol(dataCol = df[col])<jupyter_output><empty_output><jupyter_text>Train-Test Split<jupyter_code># Randomize the dataset
data_randomized = df.sample(frac=1, random_state=1)

# Calculate index for split
training_test_index = round(len(data_randomized) * 0.8)

# Split into training and test sets
training_set = data_randomized[:training_test_index].reset_index(drop=True)
test_set = data_randomized[training_test_index:].reset_index(drop=True)

print(training_set.shape)
print(test_set.shape)
training_set['Prediction'].value_counts(normalize=True)
test_set['Prediction'].value_counts(normalize=True)
# After splitting
display(training_set.head(3))
display(test_set.head(3))
## exporting test train data
training_set.to_csv("train.csv", index=False)
test_set.to_csv("test.csv", index=False)
temp = [x == 0 for x in test_set['Prediction']]
temp[11:20]<jupyter_output><empty_output><jupyter_text>---  ---  NB from sklearn<jupyter_code># not possible as plaing data is not available..

# instead we have the tocanised dataset<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10858835143139191, 'fraction_numerical': 0.005923000987166831, 'mean_word_length': 3.720496894409938, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '412110', 'license': 'MIT', 'n_tokens_mistral': 1021, 'n_tokens_neox': 994, 'n_words': 242, 'path': 'Spam Filtering/Emails/NB/emailClassification.ipynb', 'repo_name': 'ravi-prakash1907/Machine-Learning-for-Cyber-Security', 'size': '3039'}