<jupyter_start><jupyter_text>Niave Bayes Example Notebook  Notebook describes how to calculate accuracy, split datasets, create a confusion matrix, use sklearn, spark, and perform the calculation by hand with Niave Bayes with different methods.Table of Contents:- Read in data- Perform calcualtion by hand- Use sklearn- Use Spark<jupyter_code>from io import StringIO
import requests
import json
import pandas as pd



df_data_1 = pd.read_csv('../data/titanic/train.csv')
df_data_1.head()
# Niave Bayes Model by Hand!
countDict = {}
target = df_data_1['Survived']
count = df_data_1['Survived'].count()
target_countDict = target.value_counts().to_dict()
input_features = df_data_1[['Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked', 'Survived']]

# Make Model
for val in input_features['Survived'].unique():
    countDict[val]={}
    filterDF = input_features[input_features['Survived'] == val]
    for column in filterDF:
        if column != 'Survived':
            mike = filterDF[column].value_counts().to_dict()
            countDict[val][column] = mike
print("Survived (Target): ", target_countDict)
print("Input Features (Training): ", countDict)


# Predict new value
test = [2, 'female', 2, 3, 'C']
result={}
for val in target_countDict.keys():
    print("\n", val)
    result[val] = target_countDict[val] / count
    print(result[val])
    for i, item in enumerate(test):
        print(val, list(countDict[0].keys())[i]) 
        print(countDict[val][list(countDict[0].keys())[i]][item] / target_countDict[val])
        result[val] = result[val] * (countDict[val][list(countDict[0].keys())[i]][item] / target_countDict[val])
        print(result[val])

# Get max
max = ('', 0)
for r in result:
    max=(r, result[r])
print("\nPrediction:\n","  Input: ", test, "\n   Output: ", max)
import numpy as np
df_data_1['Sex'] = np.where(df_data_1['Sex'] == 'male', 0, 1)
df_data_1['Embarked'] = df_data_1['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})
df_data_1['Age'] = df_data_1['Age'].fillna(df_data_1['Age'].mean())
print(df_data_1['Embarked'].mode())
df_data_1['Embarked'] = df_data_1['Embarked'].fillna(0)
df_data_1.head()
from sklearn.naive_bayes import GaussianNB
import numpy as np
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix

#Create a Gaussian Classifier
model = GaussianNB()
# Separate into feature set and target
target = df_data_1['Survived'].values
inputData = df_data_1[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']].values

# Train & Predict (Use training data)
model.fit(inputData, target)
pred = model.predict(inputData)

#Show results
print("Number of mislabeled items out of %d: %d"
      % (inputData.shape[0],(target != pred).sum()))
print("Number of correctly labeled items out of %d: %d"
      % (inputData.shape[0],(target == pred).sum()))
cnf_matrix = confusion_matrix(target, pred)
print("Confusion Matrix:\n",cnf_matrix)
# Randomize and split
df_data_1 = df_data_1.sample(frac=1)
split = int(df_data_1['PassengerId'].count()*0.7)
test = df_data_1.iloc[split:, :]
train = df_data_1.iloc[:split, :]

# Separate into feature set and target
test_target = test['Survived'].values
test_inputData = test[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']].values
train_target = train['Survived'].values
train_inputData = train[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']].values

# Train & Predict (Use training data)
model.fit(train_inputData, train_target)
pred = model.predict(test_inputData)

# Show results
print("Number of mislabeled items out of %d: %d"
      % (test_inputData.shape[0],(test_target != pred).sum()))
print("Number of correctly labeled items out of %d: %d"
      % (test_inputData.shape[0],(test_target == pred).sum()))
cnf_matrix = confusion_matrix(test_target, pred)
print("Confusion Matrix:\n",cnf_matrix)
import pyspark
from pyspark.sql import SparkSession

spark = SparkSession.builder.appName('MG').getOrCreate()

#df = spark.read.csv("/Users/<EMAIL>/Downloads/titanic/train.csv")
df_data_2 = spark.read.format("csv").option("header", "true").load("/Users/<EMAIL>/Downloads/titanic/train.csv")

df_data_2.head(5)
from pyspark.mllib.classification import NaiveBayes, NaiveBayesModel
from pyspark.mllib.util import MLUtils
from pyspark.mllib.regression import LabeledPoint
from pyspark.sql.functions import *



# Find null/nan values
df_data_2.select([count(when(isnull(c), c)).alias(c) for c in df_data_2.columns]).show()
df_data_2.select([count(when(isnan(c), c)).alias(c) for c in df_data_2.columns]).show()

# Clean up data
data = df_data_2.withColumn('Sex', regexp_replace('Sex', 'male', '0')).withColumn('Sex', regexp_replace('Sex', 'female', '1')).withColumn('Sex', regexp_replace('Sex', 'fe0', '1'))
data = data.withColumn('Embarked', regexp_replace('Embarked', 'Q', '0')).withColumn('Embarked', regexp_replace('Embarked', 'S', '1')).withColumn('Embarked', regexp_replace('Embarked', 'C', '2'))
data = data.fillna({'Age': 29, 'Embarked': 'S'})
data = data.select("Pclass", "Sex", "Age", "SibSp", "Parch", "Embarked", "Survived").rdd
data = data.map(lambda seq: LabeledPoint(seq[-1],seq[:-2]))


# Split training (60%) / test (40%)
training, test = data.randomSplit([0.6, 0.4])

data.take(5)
# Train
model = NaiveBayes.train(training, 1.0)

# Predict and test
predictionAndLabel = test.map(lambda p: (model.predict(p.features), p.label))
accuracy = 1.0 * predictionAndLabel.filter(lambda x: x[0] == x[1]).count() / test.count()
print('Accuracy: %f' % accuracy)
TP = predictionAndLabel.filter(lambda x: x[0] == 1 and x[1] == 1).count()
FP = predictionAndLabel.filter(lambda x: x[0] == 1 and x[1] == 0).count()
TN = predictionAndLabel.filter(lambda x: x[0] == 0 and x[1] == 0).count()
FN = predictionAndLabel.filter(lambda x: x[0] == 0 and x[1] == 1).count()
print('Confusion Matrix:\n %d  | %d \n %d  | %d' % (TP, FP, FN, TN))<jupyter_output>Accuracy: 0.797844
Confusion Matrix:
 97  | 31 
 44  | 199
{'fraction_non_alphanumeric': 0.1763912310286678, 'fraction_numerical': 0.018381112984822933, 'mean_word_length': 5.582685904550499, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '243010', 'license': 'MIT', 'n_tokens_mistral': 2222, 'n_tokens_neox': 2130, 'n_words': 495, 'path': 'code/Naive Bayes.ipynb', 'repo_name': 'gildmi/data477', 'size': '5954'}