# Path: feet/utils/logger.py
# class LoggingMixin(object):
#     """
#     Mix in to classes that need their own logging object!
#     """
# 
#     @property
#     def logger(self):
#         """
#         Instantiates and returns a ServiceLogger instance
#         """
#         if not hasattr(self, '_logger') or not self._logger:
#             self._logger = ServiceLogger()
#         return self._logger
# 
# Path: feet/utils/decorators.py
# def timeit(func):
#     """
#     Returns the number of seconds that a function took along with the result
#     """
# 
#     @wraps(func)
#     def timer_wrapper(*args, **kwargs):
#         """
#         Inner function that uses the Timer context object
#         """
#         with Timer() as timer:
#             result = func(*args, **kwargs)
# 
#         return result, timer
# 
#     return timer_wrapper


from nltk import word_tokenize, sent_tokenize, ne_chunk, \
    RegexpParser, pos_tag
from nltk.tree import Tree
from collections import defaultdict
from langdetect import detect_langs
from jptools import JAParser
from feet.utils.logger import LoggingMixin
from feet.utils.decorators import timeit
import traceback
        if lang is not None:
            lang = WORLD_2_NLTK[lang]
        else:
            lang = self._lang
        if lang == 'japanese':
            return JAParser().word_tokenize(text)
        else:
            if not isinstance(text, unicode):
                text = text.decode('utf8')
            return [token.encode('utf8')
                    for token in word_tokenize(text, lang)]
            # return text.split()

    def sent_tokenize(self, text, lang=None):
        """
        Splits a text into sentences. Yields each sentence for processing.
        """
        if lang is not None:
            lang = WORLD_2_NLTK[lang]
        else:
            lang = self._lang
        if lang == 'japanese':
            for sentence in JAParser().sent_tokenize(text):
                yield sentence
        else:
            if not isinstance(text, unicode):
                text = text.decode('utf8')
            for sentence in sent_tokenize(text, lang):
                yield sentence


# Path: feet/utils/logger.py
# class LoggingMixin(object):
#     """
#     Mix in to classes that need their own logging object!
#     """
# 
#     @property
#     def logger(self):
#         """
#         Instantiates and returns a ServiceLogger instance
#         """
#         if not hasattr(self, '_logger') or not self._logger:
#             self._logger = ServiceLogger()
#         return self._logger
# 
# Path: feet/utils/decorators.py
# def timeit(func):
#     """
#     Returns the number of seconds that a function took along with the result
#     """
# 
#     @wraps(func)
#     def timer_wrapper(*args, **kwargs):
#         """
#         Inner function that uses the Timer context object
#         """
#         with Timer() as timer:
#             result = func(*args, **kwargs)
# 
#         return result, timer
# 
#     return timer_wrapper


# Path: feet/entities/nlp.py
from nltk import word_tokenize, sent_tokenize, ne_chunk, \
    RegexpParser, pos_tag
from nltk.tree import Tree
from collections import defaultdict
from langdetect import detect_langs
from jptools import JAParser
from feet.utils.logger import LoggingMixin
from feet.utils.decorators import timeit
import traceback

        if lang is not None:
            lang = WORLD_2_NLTK[lang]
        else:
            lang = self._lang
        if lang == 'japanese':
            return JAParser().word_tokenize(text)
        else:
            if not isinstance(text, unicode):
                text = text.decode('utf8')
            return [token.encode('utf8')
                    for token in word_tokenize(text, lang)]
            # return text.split()

    def sent_tokenize(self, text, lang=None):
        """
        Splits a text into sentences. Yields each sentence for processing.
        """
        if lang is not None:
            lang = WORLD_2_NLTK[lang]
        else:
            lang = self._lang
        if lang == 'japanese':
            for sentence in JAParser().sent_tokenize(text):
                yield sentence
        else:
            if not isinstance(text, unicode):
                text = text.decode('utf8')
            for sentence in sent_tokenize(text, lang):
                yield sentence


    @timeit