# Path: context/nlp/util.py
# def get_sentences(text):
#     """Return sentence list"""
#     if text.strip() == '':
#         return []
#     tokens = nltk.sent_tokenize(text)
#     sentences = []
#     for sent in tokens:
#         sentences.extend([s.strip() for s in sent.split('\n') if s.strip()])
#     sentences_ = []
#     skip_next = False
#     for i, sent in enumerate(sentences):
#         if skip_next:
#             skip_next = False
#             continue
#         if i == len(sentences):
#             sentences_.append(sent)
#             break
#         if sent.split()[-1].lower() == ('no.'):
#             try:
#                 s1 = sentences[i+1]
#                 int(s1.split()[0])
#                 sentences_.append(sent + ' ' + s1)
#                 skip_next = True
#             except ValueError:
#                 sentences_.append(sent)
#         else:
#             sentences_.append(sent)
#     assert sentences[-1] in sentences_[-1]
#     return sentences_
# 
# def pos_tag(tokens):
#     """Return POS tagged tokens"""
#     return nltk.pos_tag(tokens)
# 
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# NLTK_VERSION = nltk_major_version()
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score


from collections import defaultdict, namedtuple
from .util import get_sentences, pos_tag, tokenize, NLTK_VERSION, \
    name_parts, compare_names
import codecs
import nltk
import re
    name_list = []

    for e in entity_list:
        name_forms = sorted(e['name_forms'], key=lambda s: len(s))
        while name_forms:
            name = name_forms.pop(0)
            for i in range(len(name_forms) - 1, -1, -1):
                s = name_forms[i]
                if s.find(name) > -1:
                    del name_forms[i]
            if name not in name_list:
                name_list.append(name)
        name_list.extend([s for s in name_forms if s not in name_list])

    return name_list


def name_tokens(entity):
    entity_names = set([entity['name']] + entity['name_forms'])
    entity_name_tokens = []
    for name in entity_names:
        tokens = ALNUM_PAT.findall(name)
        for token in tokens:
            if token not in entity_name_tokens:
                entity_name_tokens.append(token)
    return entity_name_tokens


def best_matches(entity_names, objs, name_key='name'):
    obj_name_parts = name_parts([obj[name_key] for obj in objs])

# Path: context/nlp/util.py
# def get_sentences(text):
#     """Return sentence list"""
#     if text.strip() == '':
#         return []
#     tokens = nltk.sent_tokenize(text)
#     sentences = []
#     for sent in tokens:
#         sentences.extend([s.strip() for s in sent.split('\n') if s.strip()])
#     sentences_ = []
#     skip_next = False
#     for i, sent in enumerate(sentences):
#         if skip_next:
#             skip_next = False
#             continue
#         if i == len(sentences):
#             sentences_.append(sent)
#             break
#         if sent.split()[-1].lower() == ('no.'):
#             try:
#                 s1 = sentences[i+1]
#                 int(s1.split()[0])
#                 sentences_.append(sent + ' ' + s1)
#                 skip_next = True
#             except ValueError:
#                 sentences_.append(sent)
#         else:
#             sentences_.append(sent)
#     assert sentences[-1] in sentences_[-1]
#     return sentences_
# 
# def pos_tag(tokens):
#     """Return POS tagged tokens"""
#     return nltk.pos_tag(tokens)
# 
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# NLTK_VERSION = nltk_major_version()
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score


# Path: context/nlp/entities.py
from collections import defaultdict, namedtuple
from .util import get_sentences, pos_tag, tokenize, NLTK_VERSION, \
    name_parts, compare_names
import codecs
import nltk
import re

    name_list = []

    for e in entity_list:
        name_forms = sorted(e['name_forms'], key=lambda s: len(s))
        while name_forms:
            name = name_forms.pop(0)
            for i in range(len(name_forms) - 1, -1, -1):
                s = name_forms[i]
                if s.find(name) > -1:
                    del name_forms[i]
            if name not in name_list:
                name_list.append(name)
        name_list.extend([s for s in name_forms if s not in name_list])

    return name_list


def name_tokens(entity):
    entity_names = set([entity['name']] + entity['name_forms'])
    entity_name_tokens = []
    for name in entity_names:
        tokens = ALNUM_PAT.findall(name)
        for token in tokens:
            if token not in entity_name_tokens:
                entity_name_tokens.append(token)
    return entity_name_tokens


def best_matches(entity_names, objs, name_key='name'):
    obj_name_parts = name_parts([obj[name_key] for obj in objs])

    entity_name_parts = name_parts(entity_names, flat=True)