<jupyter_start><jupyter_text>https://machinelearningmastery.com/naive-bayes-classifier-scratch-python/<jupyter_code>import csv
import random
import math
def loadCsv(filename):
    lines = csv.reader(open(filename, mode = "r"))
    dataset = list(lines)
    
    for i in range(len(dataset)):
        dataset[i] = [float(x) for x in dataset[i]] # float[x] would have converted all values to floats
    return dataset
#filename = 'exampleinput.csv'
#datain = loadCsv(filename)
#print('Loaded data file {0} with {1} rows'.format(filename, len(datain)))
#datain.pop(0) # removes header - this was done when data had a header row
# Note everytime you pop, it removes the first line!!!
#dataset = datain #this was used to remove the top row<jupyter_output><empty_output><jupyter_text>Splitting the dataset<jupyter_code>def splitDataset(dataset, splitRatio):
    trainSize = int(len(dataset) * splitRatio)
    trainSet = []
    copy = list(dataset)
    while len(trainSet) < trainSize:
        index = random.randrange(len(copy))
        trainSet.append(copy.pop(index))
    return[trainSet, copy]
#splitRatio = 0.67
#train, test = splitDataset(dataset, splitRatio)
#print('Training data:', train, '\n\n\nTest data:', test)
def separateByClass(dataset):
    separated = {}
    for i in range(len(dataset)):
        vector = dataset[i]
        if (vector[-1] not in separated):
            separated[vector[-1]] = []
        separated[vector[-1]].append(vector)
    return separated
#separated = separateByClass(dataset)
#print('Separated instances:', separated) # separates between those of class where R=1, and where R=0,
# it assumes that the last value is the class (last column R)
#### Calculate mean and standard deviation of each of the attributes for a class value
def mean(numbers):
    return ((sum(numbers))/float(len(numbers)))
 
def stdev(numbers):
    avg = mean(numbers)
    variance = sum([pow(x-avg,2) for x in numbers])/(len(numbers)-1)
    return math.sqrt(variance)
#numbers = [1,2,3,4,5,6,7,8,9]
#print('Summary of:', numbers, '\nmean:', mean(numbers), '\nstandard deviation:',  stdev(numbers))
def summarize(dataset):
    summaries = [(mean(attribute), stdev(attribute)) for attribute in zip(*dataset)]
    del summaries[-1]
    return summaries
def summarizeByClass(dataset):
    separated = separateByClass(dataset)
    summaries = {}
    for classValue, instances in separated.items():
        summaries[classValue] = summarize(instances)
    return summaries
def calculateProbability(x, mean, stdev):
    exponent = math.exp(
        -(math.pow(x - mean, 2) / (2 * math.pow(stdev, 2))))
    return (1 / (math.sqrt(2*math.pi) * stdev)) * exponent
def calculateClassProbabilities(summaries, inputVector):
    probabilities = {}
    for classValue, classSummaries in summaries.items():
        probabilities[classValue] = 1
        for i in range(len(classSummaries)):
            mean, stdev = classSummaries[i]
            x = inputVector[i]
            probabilities[classValue] *= calculateProbability(x, mean, stdev)
    return probabilities
def predict(summaries, inputVector):
    probabilities = calculateClassProbabilities(summaries, inputVector)
    bestLabel, bestProb = None, -1
    for classValue, probability in probabilities.items():
        if bestLabel is None or probability > bestProb:
            bestProb = probability
            bestLabel = classValue
    return bestLabel
def getPredictions(summaries, testSet):
    predictions = []
    for i in range(len(testSet)):
        result = predict(summaries, testSet[i])
        predictions.append(result)
    return predictions
 
def getAccuracy(testSet, predictions):
    correct = 0
    for i in range(len(testSet)):
        if testSet[i][-1] == predictions[i]:
            correct += 1
    return (correct/float(len(testSet))) * 100.0
 
def rescale_column_zero_one(col):
    hi = max(col)
    lo = min(col)
    return [ (x - lo) / (hi - lo) for x in col]    
def main(rescale=False):
    filename = 'data_all_fixed.csv'
    splitRatio = 0.80
    dataset = loadCsv(filename)
    
    if rescale:
        column_data = [[], [], [], [], [], []]
        for row in dataset:
            for i in range(6):
                column_data[i].append(row[i])

        rescaled_data = [column_data[0], column_data[1], [], [], [], column_data[5]]
        for i in [2, 3, 4]: # Rescale feature columns
            rescaled_data[i] = rescale_column_zero_one(column_data[i])
        
        dataset = list(zip(*rescaled_data))
    
    trainingSet, testSet = splitDataset(dataset, splitRatio)
#     print('Split', len(dataset), 'into Test', len(trainingSet),'rows', 'and', 
#           len(testSet), 'Train rows')
    # prepare model
    summaries = summarizeByClass(trainingSet)
    # test model
    predictions = getPredictions(summaries, testSet[-1:])
    print('Test predictions', predictions)
    train_predictions = getPredictions(summaries, trainingSet)
    print('Train predictions', train_predictions)
    accuracy = getAccuracy(testSet, predictions)
    print('Accuracy:', accuracy, '%')
    return accuracy, predictions
# mean([main(True) for i in range(1000)])
# mean([main(False) for i in range(1000)])
ret = main(False)
#ret[1]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12838611217092713, 'fraction_numerical': 0.010873712323540634, 'mean_word_length': 3.6357206012378427, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '515823', 'license': 'MIT', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1607, 'n_words': 425, 'path': 'bayes/BSTS_trials/Predictor_6.ipynb', 'repo_name': 'arvindkarir/python-pandas-code', 'size': '5242'}