<commit_before>'''

Created on Dec 19,2016

'''

from numpy import *


def loadDataSet():
    postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                   ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                   ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                   ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                   ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                   ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
    classVec = [0, 1, 0, 1, 0, 1]  # 1 is abusive, 0 not
    return postingList, classVec


def createVocabList(dataLists):
    vocabListSet = set([])
    for document in dataLists:
        vocabListSet = vocabListSet | set(document)
    return list(vocabListSet)


def setOfWords2Vec(vocabList, inputSet):
    result = [0] * len(vocabList)
    for word in inputSet:
        if word in vocabList:
            result[vocabList.index(word)] = 1
    return result


def trainNB0(trainMatrix, categorys):
    numberOfDocs = len(trainMatrix)
    numberOfWords = len(trainMatrix[0])
    pAbusive = sum(categorys) / numberOfDocs

    p0nums = ones(numberOfWords)
    p1nums = ones(numberOfWords)
    p0TotalWords = 2.0
    p1TotalWords = 2.0

    for i in range(numberOfDocs):
        if categorys[i] == 1:
            p1nums += trainMatrix[i]
            p1TotalWords += sum(trainMatrix[i])<commit_msg>classify<commit_after>'''

Created on Dec 19,2016

'''

from numpy import *


def loadDataSet():
    postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                   ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                   ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                   ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                   ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                   ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
    classVec = [0, 1, 0, 1, 0, 1]  # 1 is abusive, 0 not
    return postingList, classVec


def createVocabList(dataLists):
    vocabListSet = set([])
    for document in dataLists:
        vocabListSet = vocabListSet | set(document)
    return list(vocabListSet)


def setOfWords2Vec(vocabList, inputSet):
    result = [0] * len(vocabList)
    for word in inputSet:
        if word in vocabList:
            result[vocabList.index(word)] += 1
    return result


def trainNB0(trainMatrix, categorys):
    numberOfDocs = len(trainMatrix)
    numberOfWords = len(trainMatrix[0])
    pAbusive = sum(categorys) / numberOfDocs

    p0nums = ones(numberOfWords)
    p1nums = ones(numberOfWords)
    p0TotalWords = 2.0
    p1TotalWords = 2.0

    for i in range(numberOfDocs):
        if categorys[i] == 1:
            p1nums += trainMatrix[i]
            p1TotalWords += sum(trainMatrix[i])
{'fraction_non_alphanumeric': 0.16780587833219412, 'fraction_numerical': 0.02050580997949419, 'mean_word_length': 2.7381864623243932, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2086059', 'n_tokens_mistral': 1090, 'n_tokens_neox': 1011, 'n_words': 257}