# Path: feet/entities/nlp.py
# class Parser(LoggingMixin):
#     """
#     Provides NLP tools such as language detection, tokenizer and POS tagger
#     for entities
#     """
#     _auto_lang_detect = False
#     _lang = 'en'
# 
#     def __init__(self, lang='english', auto_lang_detect=False):
#         """
#         Parser class initialisation.
#         Defines a default language attribute and decides if automatic language
#         detection is turned on/off.
#         """
#         self._auto_lang_detect = auto_lang_detect
#         self._lang = WORLD_2_NLTK[lang]
# 
#     def detect_language(self, text):
#         """
#         Provides language detection of a text.
#         Fallbacks to default language defined in class initialization or 'en'
#         """
#         try:
#             if text is None:
#                 return None
#             languages = detect_langs(text.decode('utf8'))
#             if len(languages) > 0:
#                 return languages[0].lang
#         except:
#             self.logger.warning(traceback.format_exc())
#             return self._lang or 'en'
# 
#     def word_tokenize(self, text, lang=None):
#         """
#         Tokenizes a sentence.
#         """
#         if lang is not None:
#             lang = WORLD_2_NLTK[lang]
#         else:
#             lang = self._lang
#         if lang == 'japanese':
#             return JAParser().word_tokenize(text)
#         else:
#             if not isinstance(text, unicode):
#                 text = text.decode('utf8')
#             return [token.encode('utf8')
#                     for token in word_tokenize(text, lang)]
#             # return text.split()
# 
#     def sent_tokenize(self, text, lang=None):
#         """
#         Splits a text into sentences. Yields each sentence for processing.
#         """
#         if lang is not None:
#             lang = WORLD_2_NLTK[lang]
#         else:
#             lang = self._lang
#         if lang == 'japanese':
#             for sentence in JAParser().sent_tokenize(text):
#                 yield sentence
#         else:
#             if not isinstance(text, unicode):
#                 text = text.decode('utf8')
#             for sentence in sent_tokenize(text, lang):
#                 yield sentence
# 
#     @timeit
#     def tokenize(self, text, lang=None):
#         """
#         Tokenizes a text by splitting it into sentences that will be tokenized.
#         """
#         if lang is not None:
#             lang = WORLD_2_NLTK[lang]
#         else:
#             lang = self._lang
#         output = []
#         if text is None:
#             return output
#         for sentence in self.sent_tokenize(text, lang):
#             output.append(self.word_tokenize(sentence, lang))
#         return output
# 
#     def _select_entities(self,
#                          tree,
#                          extra_classes=['NNP', 'NNPS', 'NN', 'NNS']):
#         """
#         Selects candidates of entities as chunks with specific grammatic tags.
#         """
#         entities = defaultdict(list)
#         for chunk in tree:
#             if isinstance(chunk, Tree):
#                 entities[chunk._label].append(
#                     " ".join([token[0] for token in chunk.leaves()]))
#             else:
#                 if chunk[1] in extra_classes:
#                     entities['noun_phrase_leave'].append(chunk[0])
#         return [i for key, value in dict(entities).iteritems() for i in value]
# 
#     @timeit
#     def extract_entities(self, text, grammar=None, lang=None):
#         """
#         Extract entities from text
#         """
#         entities = []
#         if lang is None:
#             lang = WORLD_2_NLTK[self.detect_language(text)]
#         else:
#             if lang in WORLD_2_NLTK.keys():
#                 lang = WORLD_2_NLTK[lang]
#             else:
#                 lang = self._lang
#         if lang == 'japanese':
#             return JAParser().extract_entities(text), lang
#         pos_sentences = [pos_tag(self.word_tokenize(sentence, lang=lang))
#                          for sentence in self.sent_tokenize(text, lang=lang)]
# 
#         if grammar is not None:
#             chunker = RegexpParser(grammar)
#             for pos_sentence in pos_sentences:
#                 tree = chunker.parse(pos_sentence)
#                 self.logger.debug(tree)
#                 entities = entities + self._select_entities(tree)
#         else:
#             for pos_sentence in pos_sentences:
#                 tree = ne_chunk(pos_sentence, binary=False)
#                 self.logger.debug(tree)
#                 entities = entities + self._select_entities(tree)
#         return entities, lang


from feet.entities.nlp import Parser
import unittest
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html



class NLPEntitiesTests(unittest.TestCase):
    def setup(self):
        pass

    def test_extract_entities(self):
        """
        Test
        """
        text = 'The United Nations (UN) is an intergovernmental organization '\
            'to promote international co-operation. A replacement for the '\
            'ineffective League of Nations, the organization was established '\
            'on 24 October 1945 after World War II in order to prevent '\
            'another such conflict. At its founding, the UN had 51 member '\
            'states; there are now 193. The headquarters of the United '\
            'Nations is in Manhattan, New York City, and experiences '\
            'extraterritoriality. Further main offices are situated in '\
            'Geneva, Nairobi, and Vienna. The organization is financed by '\
            'assessed and voluntary contributions from its member states. '\
            'Its objectives include maintaining international peace and '\
            'security, promoting human rights, fostering social and economic '\
            'development, protecting the environment, and providing '\
            'humanitarian aid in cases of famine, natural disaster, and armed '\
            'conflict.'
        # probleme with World War II not concatenated
        # we may take also NN and NNS with combination of CC and IN
        grammar = 'NE : {<NNP|NNPS|NN>*?<NNP|NNPS|JJ|NNS|NN>+}'

# Path: feet/entities/nlp.py
# class Parser(LoggingMixin):
#     """
#     Provides NLP tools such as language detection, tokenizer and POS tagger
#     for entities
#     """
#     _auto_lang_detect = False
#     _lang = 'en'
# 
#     def __init__(self, lang='english', auto_lang_detect=False):
#         """
#         Parser class initialisation.
#         Defines a default language attribute and decides if automatic language
#         detection is turned on/off.
#         """
#         self._auto_lang_detect = auto_lang_detect
#         self._lang = WORLD_2_NLTK[lang]
# 
#     def detect_language(self, text):
#         """
#         Provides language detection of a text.
#         Fallbacks to default language defined in class initialization or 'en'
#         """
#         try:
#             if text is None:
#                 return None
#             languages = detect_langs(text.decode('utf8'))
#             if len(languages) > 0:
#                 return languages[0].lang
#         except:
#             self.logger.warning(traceback.format_exc())
#             return self._lang or 'en'
# 
#     def word_tokenize(self, text, lang=None):
#         """
#         Tokenizes a sentence.
#         """
#         if lang is not None:
#             lang = WORLD_2_NLTK[lang]
#         else:
#             lang = self._lang
#         if lang == 'japanese':
#             return JAParser().word_tokenize(text)
#         else:
#             if not isinstance(text, unicode):
#                 text = text.decode('utf8')
#             return [token.encode('utf8')
#                     for token in word_tokenize(text, lang)]
#             # return text.split()
# 
#     def sent_tokenize(self, text, lang=None):
#         """
#         Splits a text into sentences. Yields each sentence for processing.
#         """
#         if lang is not None:
#             lang = WORLD_2_NLTK[lang]
#         else:
#             lang = self._lang
#         if lang == 'japanese':
#             for sentence in JAParser().sent_tokenize(text):
#                 yield sentence
#         else:
#             if not isinstance(text, unicode):
#                 text = text.decode('utf8')
#             for sentence in sent_tokenize(text, lang):
#                 yield sentence
# 
#     @timeit
#     def tokenize(self, text, lang=None):
#         """
#         Tokenizes a text by splitting it into sentences that will be tokenized.
#         """
#         if lang is not None:
#             lang = WORLD_2_NLTK[lang]
#         else:
#             lang = self._lang
#         output = []
#         if text is None:
#             return output
#         for sentence in self.sent_tokenize(text, lang):
#             output.append(self.word_tokenize(sentence, lang))
#         return output
# 
#     def _select_entities(self,
#                          tree,
#                          extra_classes=['NNP', 'NNPS', 'NN', 'NNS']):
#         """
#         Selects candidates of entities as chunks with specific grammatic tags.
#         """
#         entities = defaultdict(list)
#         for chunk in tree:
#             if isinstance(chunk, Tree):
#                 entities[chunk._label].append(
#                     " ".join([token[0] for token in chunk.leaves()]))
#             else:
#                 if chunk[1] in extra_classes:
#                     entities['noun_phrase_leave'].append(chunk[0])
#         return [i for key, value in dict(entities).iteritems() for i in value]
# 
#     @timeit
#     def extract_entities(self, text, grammar=None, lang=None):
#         """
#         Extract entities from text
#         """
#         entities = []
#         if lang is None:
#             lang = WORLD_2_NLTK[self.detect_language(text)]
#         else:
#             if lang in WORLD_2_NLTK.keys():
#                 lang = WORLD_2_NLTK[lang]
#             else:
#                 lang = self._lang
#         if lang == 'japanese':
#             return JAParser().extract_entities(text), lang
#         pos_sentences = [pos_tag(self.word_tokenize(sentence, lang=lang))
#                          for sentence in self.sent_tokenize(text, lang=lang)]
# 
#         if grammar is not None:
#             chunker = RegexpParser(grammar)
#             for pos_sentence in pos_sentences:
#                 tree = chunker.parse(pos_sentence)
#                 self.logger.debug(tree)
#                 entities = entities + self._select_entities(tree)
#         else:
#             for pos_sentence in pos_sentences:
#                 tree = ne_chunk(pos_sentence, binary=False)
#                 self.logger.debug(tree)
#                 entities = entities + self._select_entities(tree)
#         return entities, lang


# Path: tests/test_entities.py
from feet.entities.nlp import Parser
import unittest

# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html



class NLPEntitiesTests(unittest.TestCase):
    def setup(self):
        pass

    def test_extract_entities(self):
        """
        Test
        """
        text = 'The United Nations (UN) is an intergovernmental organization '\
            'to promote international co-operation. A replacement for the '\
            'ineffective League of Nations, the organization was established '\
            'on 24 October 1945 after World War II in order to prevent '\
            'another such conflict. At its founding, the UN had 51 member '\
            'states; there are now 193. The headquarters of the United '\
            'Nations is in Manhattan, New York City, and experiences '\
            'extraterritoriality. Further main offices are situated in '\
            'Geneva, Nairobi, and Vienna. The organization is financed by '\
            'assessed and voluntary contributions from its member states. '\
            'Its objectives include maintaining international peace and '\
            'security, promoting human rights, fostering social and economic '\
            'development, protecting the environment, and providing '\
            'humanitarian aid in cases of famine, natural disaster, and armed '\
            'conflict.'
        # probleme with World War II not concatenated
        # we may take also NN and NNS with combination of CC and IN
        grammar = 'NE : {<NNP|NNPS|NN>*?<NNP|NNPS|JJ|NNS|NN>+}'

        result = Parser().extract_entities(text, grammar)