# Path: context/nlp/util.py
# def get_sentences(text):
#     """Return sentence list"""
#     if text.strip() == '':
#         return []
#     tokens = nltk.sent_tokenize(text)
#     sentences = []
#     for sent in tokens:
#         sentences.extend([s.strip() for s in sent.split('\n') if s.strip()])
#     sentences_ = []
#     skip_next = False
#     for i, sent in enumerate(sentences):
#         if skip_next:
#             skip_next = False
#             continue
#         if i == len(sentences):
#             sentences_.append(sent)
#             break
#         if sent.split()[-1].lower() == ('no.'):
#             try:
#                 s1 = sentences[i+1]
#                 int(s1.split()[0])
#                 sentences_.append(sent + ' ' + s1)
#                 skip_next = True
#             except ValueError:
#                 sentences_.append(sent)
#         else:
#             sentences_.append(sent)
#     assert sentences[-1] in sentences_[-1]
#     return sentences_
# 
# def pos_tag(tokens):
#     """Return POS tagged tokens"""
#     return nltk.pos_tag(tokens)
# 
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# NLTK_VERSION = nltk_major_version()
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score


from collections import defaultdict, namedtuple
from .util import get_sentences, pos_tag, tokenize, NLTK_VERSION, \
    name_parts, compare_names
import codecs
import nltk
import re
                            entity.tokenized_name)
                        type_entity['locations'].append(locations[entity])
                else:
                    type_entities[entity] = {
                        'count': 1,
                        'name_forms': [ entity.tokenized_name ],
                        'locations': [ locations[entity] ]
                    }
        for entity_key, entity_data in type_entities.items():
            name = best_name(entity_data['name_forms'])
            if name:
                all_entities.append({
                    'name': name,
                    'type': entity_type,
                    'name_forms': [' '.join(form) for form in
                        entity_data['name_forms']],
                    'score': entity_score(
                        entity_data['count'],
                        entity_data['locations'],
                        sentences)
                })
    return all_entities


def get_entities(text):
    """Return list of entities in text sorted by score (desc)"""
    try:
        text = codecs.decode(text, 'utf-8')
    except:
        pass

# Path: context/nlp/util.py
# def get_sentences(text):
#     """Return sentence list"""
#     if text.strip() == '':
#         return []
#     tokens = nltk.sent_tokenize(text)
#     sentences = []
#     for sent in tokens:
#         sentences.extend([s.strip() for s in sent.split('\n') if s.strip()])
#     sentences_ = []
#     skip_next = False
#     for i, sent in enumerate(sentences):
#         if skip_next:
#             skip_next = False
#             continue
#         if i == len(sentences):
#             sentences_.append(sent)
#             break
#         if sent.split()[-1].lower() == ('no.'):
#             try:
#                 s1 = sentences[i+1]
#                 int(s1.split()[0])
#                 sentences_.append(sent + ' ' + s1)
#                 skip_next = True
#             except ValueError:
#                 sentences_.append(sent)
#         else:
#             sentences_.append(sent)
#     assert sentences[-1] in sentences_[-1]
#     return sentences_
# 
# def pos_tag(tokens):
#     """Return POS tagged tokens"""
#     return nltk.pos_tag(tokens)
# 
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# NLTK_VERSION = nltk_major_version()
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score


# Path: context/nlp/entities.py
from collections import defaultdict, namedtuple
from .util import get_sentences, pos_tag, tokenize, NLTK_VERSION, \
    name_parts, compare_names
import codecs
import nltk
import re

                            entity.tokenized_name)
                        type_entity['locations'].append(locations[entity])
                else:
                    type_entities[entity] = {
                        'count': 1,
                        'name_forms': [ entity.tokenized_name ],
                        'locations': [ locations[entity] ]
                    }
        for entity_key, entity_data in type_entities.items():
            name = best_name(entity_data['name_forms'])
            if name:
                all_entities.append({
                    'name': name,
                    'type': entity_type,
                    'name_forms': [' '.join(form) for form in
                        entity_data['name_forms']],
                    'score': entity_score(
                        entity_data['count'],
                        entity_data['locations'],
                        sentences)
                })
    return all_entities


def get_entities(text):
    """Return list of entities in text sorted by score (desc)"""
    try:
        text = codecs.decode(text, 'utf-8')
    except:
        pass

    sentences = get_sentences(text)