<commit_before># Natural Language Toolkit: Interface to Megam Classifier
#
# Copyright (C) 2008 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org>
# For license information, see LICENSE.TXT
#
# $Id: naivebayes.py 2063 2004-07-17 21:02:24Z edloper $

"""
A set of functions used to interface with the external U{megam
<http://www.cs.utah.edu/~hal/megam/>} maxent optimization package.
Before C{megam} can be used, you should tell NLTK where it can find
the C{megam} binary, using the L{config_megam()} function.  Typical
usage:

    >>> import nltk
    >>> nltk.config_megam('.../path/to/megam')
    >>> classifier = nltk.MaxentClassifier.train(corpus, 'megam')

"""
__docformat__ = 'epytext en'

import os
import os.path
import subprocess

from nltk.internals import find_binary
try:
    import numpy
except ImportError:
    numpy = None

######################################################################
#{ Configuration
######################################################################

_megam_bin = None
def config_megam(bin=None):
    """
    Configure NLTK's interface to the C{megam} maxent optimization
    package.

    @param bin: The full path to the C{megam} binary.  If not specified,
        then nltk will search the system for a C{megam} binary; and if
        one is not found, it will raise a C{LookupError} exception.
    @type bin: C{string}
    """
    global _megam_bin
    _megam_bin = find_binary(
        'megam', bin,
        env_vars=['MEGAM',  'MEGAM_HOME'],
        binary_names=['megam', 'megam_686o', 'megam_i686.opto'],
        url='http://www.cs.utah.edu/~hal/megam/')

######################################################################
#{ Megam Interface Functions
######################################################################

def write_megam_file(train_toks, encoding, stream,
                     bernoulli=True, explicit=True):
    """
    Generate an input file for C{megam} based on the given corpus of
    classified tokens.

    @type train_toks: C{list} of C{tuples} of (C{dict}, C{str})
    @param train_toks: Training data, represented as a list of
        pairs, the first member of which is a feature dictionary,
        and the second of which is a classification label.

    @type encoding: L{MaxentFeatureEncodingI}
    @param encoding: A feature encoding, used to convert featuresets
        into feature vectors.

    @type stream: C{stream}
    @param stream: The stream to which the megam input file should be
        written.

    @param bernoulli: If true, then use the 'bernoulli' format.  I.e.,
        all joint features have binary values, and are listed iff they
        are true.  Otherwise, list feature values explicitly.  If
        C{bernoulli=False}, then you must call C{megam} with the
        C{-fvals} option.

    @param explicit: If true, then use the 'explicit' format.  I.e.,
        list the features that would fire for any of the possible
        labels, for each token.  If C{explicit=True}, then you must
        call C{megam} with the C{-explicit} option.
    """
    # Look up the set of labels.
    labels = encoding.labels()
    labelnum = dict([(label, i) for (i, label) in enumerate(labels)])

    # Write the file, which contains one line per instance.
    for featureset, label in train_toks:
        # First, the instance number.
        stream.write('%d' % labelnum[label])

        # For implicit file formats, just list the features that fire
        # for thie instance's actual label.
        if not explicit:
            _write_megam_features(encoding.encode(featureset, label),
                                  stream, bernoulli)
<commit_msg>Added megam.opt to binary_names list.<commit_after># Natural Language Toolkit: Interface to Megam Classifier
#
# Copyright (C) 2008 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org>
# For license information, see LICENSE.TXT
#
# $Id: naivebayes.py 2063 2004-07-17 21:02:24Z edloper $

"""
A set of functions used to interface with the external U{megam
<http://www.cs.utah.edu/~hal/megam/>} maxent optimization package.
Before C{megam} can be used, you should tell NLTK where it can find
the C{megam} binary, using the L{config_megam()} function.  Typical
usage:

    >>> import nltk
    >>> nltk.config_megam('.../path/to/megam')
    >>> classifier = nltk.MaxentClassifier.train(corpus, 'megam')

"""
__docformat__ = 'epytext en'

import os
import os.path
import subprocess

from nltk.internals import find_binary
try:
    import numpy
except ImportError:
    numpy = None

######################################################################
#{ Configuration
######################################################################

_megam_bin = None
def config_megam(bin=None):
    """
    Configure NLTK's interface to the C{megam} maxent optimization
    package.

    @param bin: The full path to the C{megam} binary.  If not specified,
        then nltk will search the system for a C{megam} binary; and if
        one is not found, it will raise a C{LookupError} exception.
    @type bin: C{string}
    """
    global _megam_bin
    _megam_bin = find_binary(
        'megam', bin,
        env_vars=['MEGAM',  'MEGAM_HOME'],
        binary_names=['megam.opt', 'megam', 'megam_686o', 'megam_i686.opto'],
        url='http://www.cs.utah.edu/~hal/megam/')

######################################################################
#{ Megam Interface Functions
######################################################################

def write_megam_file(train_toks, encoding, stream,
                     bernoulli=True, explicit=True):
    """
    Generate an input file for C{megam} based on the given corpus of
    classified tokens.

    @type train_toks: C{list} of C{tuples} of (C{dict}, C{str})
    @param train_toks: Training data, represented as a list of
        pairs, the first member of which is a feature dictionary,
        and the second of which is a classification label.

    @type encoding: L{MaxentFeatureEncodingI}
    @param encoding: A feature encoding, used to convert featuresets
        into feature vectors.

    @type stream: C{stream}
    @param stream: The stream to which the megam input file should be
        written.

    @param bernoulli: If true, then use the 'bernoulli' format.  I.e.,
        all joint features have binary values, and are listed iff they
        are true.  Otherwise, list feature values explicitly.  If
        C{bernoulli=False}, then you must call C{megam} with the
        C{-fvals} option.

    @param explicit: If true, then use the 'explicit' format.  I.e.,
        list the features that would fire for any of the possible
        labels, for each token.  If C{explicit=True}, then you must
        call C{megam} with the C{-explicit} option.
    """
    # Look up the set of labels.
    labels = encoding.labels()
    labelnum = dict([(label, i) for (i, label) in enumerate(labels)])

    # Write the file, which contains one line per instance.
    for featureset, label in train_toks:
        # First, the instance number.
        stream.write('%d' % labelnum[label])

        # For implicit file formats, just list the features that fire
        # for this instance's actual label.
        if not explicit:
            _write_megam_features(encoding.encode(featureset, label),
                                  stream, bernoulli)

{'fraction_non_alphanumeric': 0.16623798565046705, 'fraction_numerical': 0.007580885339109246, 'mean_word_length': 3.2435381964388283, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5249500', 'n_tokens_mistral': 2284, 'n_tokens_neox': 2128, 'n_words': 761}