<commit_before># Natural Language Toolkit: Naive Bayes Classifiers
#
# Copyright (C) 2001 University of Pennsylvania
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

from nltk.classifier import *
from nltk.probability import FreqDistI
from nltk.probability import SimpleFreqDist
from nltk.probability import CFFreqDist, CFSample, ContextEvent
from nltk.probability import MLEProbDist, ELEProbDist
from nltk.probability import ProbDistI, EventI
from nltk.token import Token, WSTokenizer
from nltk.chktype import chktype as _chktype

import time
from Numeric import zeros

##//////////////////////////////////////////////////////
##  Labeled FeatureValueList & Events
##//////////////////////////////////////////////////////

class LabeledFeatureValueList:
    def __init__(self, fvl, label):
        self._fvl = fvl
        self._label = label
    def feature_values(self):
        return self._fvl
    def label(self):
        return self._label
    def __repr__(self):
        return '<LabeledFeatureValueList %r %r>' % (self._label, self._fvl) 

class LabelEvent(EventI):
    def __init__(self, label):
        self._label = label
    def contains(self, sample):
        return sample._label == self._label
    def label(self):
        return self._label
    def __repr__(self):
        return ('{Event x: x=LabeledFeatureValueList' +
                ' with label %r}' % self._label)

class AssignmentEvent(EventI):
    def __init__(self, assignment):
        self._assignment = assignment
    def assignment(self):
        return self._assignment
    def contains(self, sample):
        return sample._assignment == self._assignment
    def __repr__(self):
        return ('{Event x: x=LabeledFeatureValueList' +
                ' with assignment %r}' % self._assignment)

##//////////////////////////////////////////////////////
##  Specialized frequency distribution.
##//////////////////////////////////////////////////////

class NBClassifierFreqDist(FreqDistI):
    """
    A frequency distribution whose samples are labeled feature value
    lists; and which is specialized for finding the following
    frequencies:
    
        - freq(label)
        - freq(assignment|label)

    Assumes binary features (for now).

    Internally, maintain a 2d array of features x labels, which we use
    to count the number of times each feature is active with a label.
    Use a single global N.
    
    """
    def __init__(self, labels, features):
        self._features = features
        self._num_features = len(features)
        self._labels = labels
        
        # Map from labels -> indices.
        self._num_labels = len(labels)
        self._label_map = {}
        i = 0
        for label in labels:
            self._label_map[label] = i
            i += 1

        # Keep two arrays..
        self._fcounts = zeros( (self._num_labels, self._num_features) )
        self._lcounts = zeros(self._num_labels)
        self._N = 0.0

        # Cache...
        self._ffreqs = None
        self._lfreqs = None

    def B(self):
        #?!?!?
        return self._num_labels * (2L**self._num_features)

    def N(self):
        return self._N

    def inc(self, sample):
        _chktype('NBClassifierFreqDist.inc', 1, sample, (LabeledFeatureValueList,))

        # Increment the total count.
        self._N += 1

        # Get the label index.
        label_index = self._label_map[sample.label()]

        # Increment the label count array
        self._lcounts[label_index] += 1

        # Increment the feature count array
        for (feature_id,val) in sample.feature_values().assignments():
            self._fcounts[label_index, feature_id] += 1

    def freq(self, event):
        # EFFICIENCY TEST:
        #return 0
        
        _chktype('NBClassifierFreqDist.freq', 1, event, (LabelEvent,))
        label_index = self._label_map[event.label()]
        return self._lcounts[label_index] / self._N

    def cond_freq(self, event, condition):
        # EFFICIENCY TEST:
        #return 0
        
        #_chktype('NBClassifierFreqDist.cond_freq', 1, event, (AssignmentEvent,))
        #_chktype('NBClassifierFreqDist.cond_freq', 2, condition, (LabelEvent,))
        label_index = self._label_map[condition.label()]
        (feature_id, value) = event.assignment()

        if value:
            return self._fcounts[label_index, feature_id] / self._N
        else:
            return 1 - (self._fcounts[label_index, feature_id] / self._N)

    def cheat(self, text):
        if self._ffreqs is None: self._ffreqs = self._fcounts / self._N
        if self._lfreqs is None: self._lfreqs = self._lcounts / self._N
        
        # Convert the feature values into an array.
        fvals = zeros( (self._num_labels, self._num_features) )
        for label_id in range(self._num_labels):
            label = self._labels[label_id]
            values = self._features.detect(LabeledText(text, label))
            for (feature_id, val) in values.assignments():
                fvals[label_id][feature_id] = val

        # Copy it num_labels times.
        fvals = fvals * ones( (self._num_labels, 1) )

        # Do element-wise multiplication
        fplus = self._ffreqs * fvals
        fminus = (1-self._ffreqs) * (1-fvals)
        ftot = fplus + fminus

        fprobs = product(transpose(ftot))
        probs = fprobs * self._lfreqs

        return self._labels[argmax(probs)]


##//////////////////////////////////////////////////////
##  Naive Bayes Classifier
##//////////////////////////////////////////////////////

class NBClassifier(ClassifierI):
    """
    A Naive Bayes classifier::

      P(inst, label) = P(label) * P(fi|label) * (1-P(fj|label))

    Where fi are the features that fired; and fj are the features that
    did not fire.

    Model parameters:
      - FeatureDetectorList
      - A PDF whose samples are C{LabeledFeatureValueList}s
      - PDF for feature|label

    Hm.. So maybe have a different NBProbDist for each label..
    
    """
    def __init__(self, featuredetectors, labels, pdist):
        self._featuredetectors = featuredetectors
        self._labels = labels
        self._pdist = pdist

    def prob(self, labeled_token):
        """
        Find the probability of the given labeled instance, using the
        naive bayes assumption.
        """
        labeled_text = labeled_token.type()
        label = labeled_text.label()
        text = labeled_text.text()
        
        p = self._pdist.prob(LabelEvent(label))
        featurevalues = self._featuredetectors.detect(labeled_text)
        #for feature_id in range(len(featurevalues)):
        #    assignment = (feature_id, featurevalues[feature_id])
        #    p *= self._pdist.cond_prob(AssignmentEvent(assignment),
        #                               LabelEvent(label))
        for assignment in featurevalues.assignments():
            p *= self._pdist.cond_prob(AssignmentEvent(assignment),
                                       LabelEvent(label))

        return p<commit_msg>- Moved LabeledFeatureValueList from naivebayes.py to __init__.py - Added module docstring & assorted class docstrings<commit_after># Natural Language Toolkit: Naive Bayes Classifiers
#
# Copyright (C) 2001 University of Pennsylvania
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

"""
A text classifier model based on the naive bayes assumption.  
In particular, this classifier assumes that the probability of each
feature occuring is independant, given the label.  This allows us to
estimate the probability that a given text has a given label as
follows::

      P(label|text) = P(text,label) / P(text)

      P(text, label) = P(label) * P(fi|label) * (1-P(fj|label))

Where C{text} is the text we are classifiying; C{label} is a potential
label; C{f1}, C{f2}, ..., C{fi}, ..., C{fN} are the 

blah blah..

Approach:
  - define useful events
  - NBClassifier gets a single PDF over LabeledFeatureValueLists.

"""

from nltk.classifier import *
from nltk.probability import FreqDistI
from nltk.probability import SimpleFreqDist
from nltk.probability import CFFreqDist, CFSample, ContextEvent
from nltk.probability import MLEProbDist, ELEProbDist
from nltk.probability import ProbDistI, EventI
from nltk.token import Token, WSTokenizer
from nltk.chktype import chktype as _chktype

import time
from Numeric import zeros

##//////////////////////////////////////////////////////
##  Events for LabeledFeatureValueList
##//////////////////////////////////////////////////////

class LabelEvent(EventI):
    """
    An event defined over C{LabeledFeatureValueList}s which is true
    when the C{LabeledFeatureValueList}'s label has a given value.
    """
    def __init__(self, label):
        self._label = label
    def contains(self, sample):
        return sample._label == self._label
    def label(self):
        return self._label
    def __repr__(self):
        return ('{Event x: x=LabeledFeatureValueList' +
                ' with label %r}' % self._label)

class AssignmentEvent(EventI):
    """
    An event defined over C{LabeledFeatureValueList}s which is true
    when the C{LabeledFeatureValueList}'s feature value list contains
    a given assignment.  An assignment is an (id, value) pair which
    gives a feature identifier and the value of that feature.
    """
    def __init__(self, assignment):
        """
        Construct a new C{AssignmentEvent}, which is true for any
        C{LabeledFeatureValueList} whose feature value list contains
        the given assignment.

        @param assignment: The assignment to check for.  This (id,
            value) pair specifies a feature's identifier and a value
            for that feature.  For Naive Bayes classifiers, values are
            treated as binary values.
        @type assignment: C{tuple} of C{int} and (immutable)
        """
        self._assignment = assignment
        
    def assignment(self):
        return self._assignment
    def contains(self, sample):
        return sample._assignment == self._assignment
    def __repr__(self):
        return ('{Event x: x=LabeledFeatureValueList' +
                ' with assignment %r}' % self._assignment)

##//////////////////////////////////////////////////////
##  Specialized frequency distribution.
##//////////////////////////////////////////////////////

class NBClassifierFreqDist(FreqDistI):
    """
    A frequency distribution whose samples are labeled feature value
    lists; and which is specialized for finding the following
    frequencies:
    
        - freq(label)
        - freq(assignment|label)

    Assumes binary features (for now).

    Internally, maintain a 2d array of features x labels, which we use
    to count the number of times each feature is active with a label.
    Use a single global N.
    
    """
    def __init__(self, labels, features):
        self._features = features
        self._num_features = len(features)
        self._labels = labels
        
        # Map from labels -> indices.
        self._num_labels = len(labels)
        self._label_map = {}
        i = 0
        for label in labels:
            self._label_map[label] = i
            i += 1

        # Keep two arrays..
        self._fcounts = zeros( (self._num_labels, self._num_features) )
        self._lcounts = zeros(self._num_labels)
        self._N = 0.0

        # Cache...
        self._ffreqs = None
        self._lfreqs = None

    def B(self):
        #?!?!?
        return self._num_labels * (2L**self._num_features)

    def N(self):
        return self._N

    def inc(self, sample):
        _chktype('NBClassifierFreqDist.inc', 1, sample, (LabeledFeatureValueList,))

        # Increment the total count.
        self._N += 1

        # Get the label index.
        label_index = self._label_map[sample.label()]

        # Increment the label count array
        self._lcounts[label_index] += 1

        # Increment the feature count array
        for (feature_id,val) in sample.feature_values().assignments():
            self._fcounts[label_index, feature_id] += 1

    def freq(self, event):
        # EFFICIENCY TEST:
        #return 0
        
        _chktype('NBClassifierFreqDist.freq', 1, event, (LabelEvent,))
        label_index = self._label_map[event.label()]
        return self._lcounts[label_index] / self._N

    def cond_freq(self, event, condition):
        # EFFICIENCY TEST:
        #return 0
        
        #_chktype('NBClassifierFreqDist.cond_freq', 1, event, (AssignmentEvent,))
        #_chktype('NBClassifierFreqDist.cond_freq', 2, condition, (LabelEvent,))
        label_index = self._label_map[condition.label()]
        (feature_id, value) = event.assignment()

        if value:
            return self._fcounts[label_index, feature_id] / self._N
        else:
            return 1 - (self._fcounts[label_index, feature_id] / self._N)

    def cheat(self, text):
        if self._ffreqs is None: self._ffreqs = self._fcounts / self._N
        if self._lfreqs is None: self._lfreqs = self._lcounts / self._N
        
        # Convert the feature values into an array.
        fvals = zeros( (self._num_labels, self._num_features) )
        for label_id in range(self._num_labels):
            label = self._labels[label_id]
            values = self._features.detect(LabeledText(text, label))
            for (feature_id, val) in values.assignments():
                fvals[label_id][feature_id] = val

        # Copy it num_labels times.
        fvals = fvals * ones( (self._num_labels, 1) )

        # Do element-wise multiplication
        fplus = self._ffreqs * fvals
        fminus = (1-self._ffreqs) * (1-fvals)
        ftot = fplus + fminus

        fprobs = product(transpose(ftot))
        probs = fprobs * self._lfreqs

        return self._labels[argmax(probs)]


##//////////////////////////////////////////////////////
##  Naive Bayes Classifier
##//////////////////////////////////////////////////////

class NBClassifier(ClassifierI):
    """
    A Naive Bayes classifier::

      P(inst, label) = P(label) * P(fi|label) * (1-P(fj|label))

    Where fi are the features that fired; and fj are the features that
    did not fire.

    Model parameters:
      - FeatureDetectorList
      - A list of labels
      - A PDF whose samples are C{LabeledFeatureValueList}s
    """
    def __init__(self, featuredetectors, labels, pdist):
        self._featuredetectors = featuredetectors
        self._labels = labels
        self._pdist = pdist

    def prob(self, labeled_token):
        """
        Find the probability of the given labeled instance, using the
        naive bayes assumption.
        """
        labeled_text = labeled_token.type()
        label = labeled_text.label()
        text = labeled_text.text()
        
        p = self._pdist.prob(LabelEvent(label))
        featurevalues = self._featuredetectors.detect(labeled_text)
        #for feature_id in range(len(featurevalues)):
        #    assignment = (feature_id, featurevalues[feature_id])
        #    p *= self._pdist.cond_prob(AssignmentEvent(assignment),
        #                               LabelEvent(label))
        for assignment in featurevalues.assignments():
            p *= self._pdist.cond_prob(AssignmentEvent(assignment),
                                       LabelEvent(label))

        return p
{'fraction_non_alphanumeric': 0.13253012048192772, 'fraction_numerical': 0.003303536727555383, 'mean_word_length': 2.1297384958443137, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5916780', 'n_tokens_mistral': 4470, 'n_tokens_neox': 4353, 'n_words': 1230}