# Path: lingtools/prob/ngram.py
# class NgramModel(object):
#     """A simple N-gram model."""
# 
#     def __init__(self, n, training_data=None):
#         """Create an n order model using training_data."""
#         # Set n and train
#         self._order = n
#         self._cfd = None
#         self.reset()
# 
#         if training_data:
#             self.update(training_data)
# 
#     def reset(self):
#         """Clear all trained data structures."""
#         self._cfd = ConditionalFreqDist()
# 
#     @property
#     def order(self):
#         """Return the order (1 = unigram, 2 = bigram, etc.)  of the model."""
#         return self._order
# 
#     def update(self, training_data):
#         """Train on iterable training data."""
#         if self._order == 1:
#             # In the unigram case, the context is always None
#             train_iter = product((None,), training_data)
#         else:
#             # Skip the first n-1 events
#             events = islice(training_data, self._order - 1, None)
#             # Each iterable in context represents the trailing
#             # history. For example, the first context starts from
#             # index zero of the data, the second starts from index 1,
#             # etc. Note that the number of things in contexts is
#             # O(self._order), so making it a list instead of a generator
#             # is not a problem.
#             contexts = [islice(training_data, offset, None)
#                         for offset in range(self._order - 1)]
#             # Pair each event with each trailing context, thus for an
#             # event with index i ((..., i-2, i-1), i).
#             train_iter = izip(izip(*contexts), events)
#         # Update on the data
#         for context, event in train_iter:
#             self._cfd[context].inc(event)
# 
#     def prob(self, event, context, smoothing=Smoothing.NONE):
#         """Return the probability for an event in the provided context"""
#         # Sanitize context since it may accidentally be a list
#         if context is not None:
#             context = tuple(context)
# 
#         # Explicitly test for presence of the context, as it will be
#         # automatically created if we try to get it.
#         if context not in self._cfd:
#             # Unknown contexts are not smoothed. It's pretty hard to
#             # figure out how you would smooth them anyway; that's what
#             # backoff is for. It's up to the caller to decide what to
#             # do in this scenario.
#             raise NoSuchContextException
# 
#         estimator = self._cfd[context]
# 
#         if smoothing:
#             if smoothing == Smoothing.LAPLACE:
#                 add = 1
#             elif smoothing == Smoothing.ELE:
#                 add = 0.5
#             else:
#                 raise ValueError("Uknown smoothing: {}".format(smoothing))
#             count = estimator.count(event)
#             total_count = estimator.total_count
#             event_count = estimator.total_outcomes
#             return (count + add) / (total_count + (event_count * add))
#         else:
#             return estimator.freq(event)
# 
#     def count(self, event, context):
#         """Return the count for an event in the provided context."""
#         # Sanitize context since it may accidentally be a list
#         if context is not None:
#             context = tuple(context)
#         try:
#             estimator = self._cfd[context]
#         except KeyError:
#             return 0
# 
#         return estimator.count(event)
# 
#     def context_count(self, context):
#         """Return the total count for a given context."""
#         # Sanitize context since it may accidentally be a list
#         if context is not None:
#             context = tuple(context)
#         try:
#             estimator = self._cfd[context]
#         except KeyError:
#             return 0
# 
#         return estimator.total_count
# 
#     def allngrams(self):
#         """Return all N-grams observed by the model and their probabilities."""
#         ngram_probs = ((event, context, self.prob(event, context))
#                        for context, dist in self._cfd.items()
#                        for event in dist)
#         return sorted(ngram_probs, key=itemgetter(1))
# 
# class NoSuchContextException(Exception):
#     """Raised when information is requested from an unobserved context."""
#     pass


import unittest
from collections import Counter
from nltk import FreqDist, ConditionalFreqDist
from lingtools.prob.ngram import NgramModel, NoSuchContextException
    """The celebration had a long way to go and even in the silent depths of
Multivac's underground chambers, it hung in the air.
If nothing else, there was the mere fact of isolation and silence. For
the first time in a decade, technicians were not scurrying about the
vitals of the giant computer, the soft lights did not wink out their
erratic patterns, the flow of information in and out had halted.
It would not be halted long, of course, for the needs of peace would be
pressing. Yet now, for a day, perhaps for a week, even Multivac might
celebrate the great time, and rest.
Lamar Swift took off the military cap he was wearing and looked down
the long and empty main corridor of the enormous computer. He sat down
rather wearily in one of the technician's swing-stools, and his
uniform, in which he had never been comfortable, took on a heavy and
wrinkled appearance.
He said, "I'll miss it all after a grisly fashion. It's hard to
remember when we weren't at war with Deneb, and it seems against nature
now to be at peace and to look at the stars without anxiety."
The two men with the Executive Director of the Solar Federation were
both younger than Swift. Neither was as gray. Neither looked quite as
tired."""
TEST_TOKENS = [token.lower() for token in
               TEST_PASSAGE.replace("\n", " ").translate(None, '",.').split(" ")]


class TestBasic(unittest.TestCase):
    """Test non-probability functions."""

    def test_order(self):
        """Be able to initialize up to some reasonable N."""
        for order in range(10):

# Path: lingtools/prob/ngram.py
# class NgramModel(object):
#     """A simple N-gram model."""
# 
#     def __init__(self, n, training_data=None):
#         """Create an n order model using training_data."""
#         # Set n and train
#         self._order = n
#         self._cfd = None
#         self.reset()
# 
#         if training_data:
#             self.update(training_data)
# 
#     def reset(self):
#         """Clear all trained data structures."""
#         self._cfd = ConditionalFreqDist()
# 
#     @property
#     def order(self):
#         """Return the order (1 = unigram, 2 = bigram, etc.)  of the model."""
#         return self._order
# 
#     def update(self, training_data):
#         """Train on iterable training data."""
#         if self._order == 1:
#             # In the unigram case, the context is always None
#             train_iter = product((None,), training_data)
#         else:
#             # Skip the first n-1 events
#             events = islice(training_data, self._order - 1, None)
#             # Each iterable in context represents the trailing
#             # history. For example, the first context starts from
#             # index zero of the data, the second starts from index 1,
#             # etc. Note that the number of things in contexts is
#             # O(self._order), so making it a list instead of a generator
#             # is not a problem.
#             contexts = [islice(training_data, offset, None)
#                         for offset in range(self._order - 1)]
#             # Pair each event with each trailing context, thus for an
#             # event with index i ((..., i-2, i-1), i).
#             train_iter = izip(izip(*contexts), events)
#         # Update on the data
#         for context, event in train_iter:
#             self._cfd[context].inc(event)
# 
#     def prob(self, event, context, smoothing=Smoothing.NONE):
#         """Return the probability for an event in the provided context"""
#         # Sanitize context since it may accidentally be a list
#         if context is not None:
#             context = tuple(context)
# 
#         # Explicitly test for presence of the context, as it will be
#         # automatically created if we try to get it.
#         if context not in self._cfd:
#             # Unknown contexts are not smoothed. It's pretty hard to
#             # figure out how you would smooth them anyway; that's what
#             # backoff is for. It's up to the caller to decide what to
#             # do in this scenario.
#             raise NoSuchContextException
# 
#         estimator = self._cfd[context]
# 
#         if smoothing:
#             if smoothing == Smoothing.LAPLACE:
#                 add = 1
#             elif smoothing == Smoothing.ELE:
#                 add = 0.5
#             else:
#                 raise ValueError("Uknown smoothing: {}".format(smoothing))
#             count = estimator.count(event)
#             total_count = estimator.total_count
#             event_count = estimator.total_outcomes
#             return (count + add) / (total_count + (event_count * add))
#         else:
#             return estimator.freq(event)
# 
#     def count(self, event, context):
#         """Return the count for an event in the provided context."""
#         # Sanitize context since it may accidentally be a list
#         if context is not None:
#             context = tuple(context)
#         try:
#             estimator = self._cfd[context]
#         except KeyError:
#             return 0
# 
#         return estimator.count(event)
# 
#     def context_count(self, context):
#         """Return the total count for a given context."""
#         # Sanitize context since it may accidentally be a list
#         if context is not None:
#             context = tuple(context)
#         try:
#             estimator = self._cfd[context]
#         except KeyError:
#             return 0
# 
#         return estimator.total_count
# 
#     def allngrams(self):
#         """Return all N-grams observed by the model and their probabilities."""
#         ngram_probs = ((event, context, self.prob(event, context))
#                        for context, dist in self._cfd.items()
#                        for event in dist)
#         return sorted(ngram_probs, key=itemgetter(1))
# 
# class NoSuchContextException(Exception):
#     """Raised when information is requested from an unobserved context."""
#     pass


# Path: lingtools/prob/test_ngram.py
import unittest
from collections import Counter
from nltk import FreqDist, ConditionalFreqDist
from lingtools.prob.ngram import NgramModel, NoSuchContextException

    """The celebration had a long way to go and even in the silent depths of
Multivac's underground chambers, it hung in the air.
If nothing else, there was the mere fact of isolation and silence. For
the first time in a decade, technicians were not scurrying about the
vitals of the giant computer, the soft lights did not wink out their
erratic patterns, the flow of information in and out had halted.
It would not be halted long, of course, for the needs of peace would be
pressing. Yet now, for a day, perhaps for a week, even Multivac might
celebrate the great time, and rest.
Lamar Swift took off the military cap he was wearing and looked down
the long and empty main corridor of the enormous computer. He sat down
rather wearily in one of the technician's swing-stools, and his
uniform, in which he had never been comfortable, took on a heavy and
wrinkled appearance.
He said, "I'll miss it all after a grisly fashion. It's hard to
remember when we weren't at war with Deneb, and it seems against nature
now to be at peace and to look at the stars without anxiety."
The two men with the Executive Director of the Solar Federation were
both younger than Swift. Neither was as gray. Neither looked quite as
tired."""
TEST_TOKENS = [token.lower() for token in
               TEST_PASSAGE.replace("\n", " ").translate(None, '",.').split(" ")]


class TestBasic(unittest.TestCase):
    """Test non-probability functions."""

    def test_order(self):
        """Be able to initialize up to some reasonable N."""
        for order in range(10):

            model = NgramModel(order)