# Path: context/nlp/util.py
# def get_sentences(text):
#     """Return sentence list"""
#     if text.strip() == '':
#         return []
#     tokens = nltk.sent_tokenize(text)
#     sentences = []
#     for sent in tokens:
#         sentences.extend([s.strip() for s in sent.split('\n') if s.strip()])
#     sentences_ = []
#     skip_next = False
#     for i, sent in enumerate(sentences):
#         if skip_next:
#             skip_next = False
#             continue
#         if i == len(sentences):
#             sentences_.append(sent)
#             break
#         if sent.split()[-1].lower() == ('no.'):
#             try:
#                 s1 = sentences[i+1]
#                 int(s1.split()[0])
#                 sentences_.append(sent + ' ' + s1)
#                 skip_next = True
#             except ValueError:
#                 sentences_.append(sent)
#         else:
#             sentences_.append(sent)
#     assert sentences[-1] in sentences_[-1]
#     return sentences_
# 
# def pos_tag(tokens):
#     """Return POS tagged tokens"""
#     return nltk.pos_tag(tokens)
# 
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# NLTK_VERSION = nltk_major_version()
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score


from collections import defaultdict, namedtuple
from .util import get_sentences, pos_tag, tokenize, NLTK_VERSION, \
    name_parts, compare_names
import codecs
import nltk
import re
                    if NLTK_VERSION >= 3:
                        c.set_label('LOCATION')
                    else:
                        c.node = 'LOCATION'
            yield c
        else:
            yield chunk


def _entity_types_and_locations(sentences):
    entities_by_type = defaultdict(list)
    locations = {}
    for sentnum, sent in enumerate(sentences, start=1):
        sent = pos_tag(tokenize(sent))
        chunks = chunk_named_entities(sent)
        chunks = [chunk for chunk in retag(chunks)]
        parser = nltk.RegexpParser(entity_chunk_grammar)
        parsed = parser.parse(chunks)
        for chunk in parsed:
            if not isinstance(chunk, nltk.tree.Tree):
                continue
            tokenized_name = tuple([c[0] for c in chunk.leaves()])
            e = KNOWN_ENTITIES.get(tokenized_name)
            if e:
                entity_key = EntityKey(tokenized_name=tokenized_name,
                    normalized_name=e['name'])
                entities_by_type[e['type']].append(entity_key)
            else:
                entity_key = EntityKey(tokenized_name=tokenized_name,
                    normalized_name=normalize_name(tokenized_name))

# Path: context/nlp/util.py
# def get_sentences(text):
#     """Return sentence list"""
#     if text.strip() == '':
#         return []
#     tokens = nltk.sent_tokenize(text)
#     sentences = []
#     for sent in tokens:
#         sentences.extend([s.strip() for s in sent.split('\n') if s.strip()])
#     sentences_ = []
#     skip_next = False
#     for i, sent in enumerate(sentences):
#         if skip_next:
#             skip_next = False
#             continue
#         if i == len(sentences):
#             sentences_.append(sent)
#             break
#         if sent.split()[-1].lower() == ('no.'):
#             try:
#                 s1 = sentences[i+1]
#                 int(s1.split()[0])
#                 sentences_.append(sent + ' ' + s1)
#                 skip_next = True
#             except ValueError:
#                 sentences_.append(sent)
#         else:
#             sentences_.append(sent)
#     assert sentences[-1] in sentences_[-1]
#     return sentences_
# 
# def pos_tag(tokens):
#     """Return POS tagged tokens"""
#     return nltk.pos_tag(tokens)
# 
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# NLTK_VERSION = nltk_major_version()
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score


# Path: context/nlp/entities.py
from collections import defaultdict, namedtuple
from .util import get_sentences, pos_tag, tokenize, NLTK_VERSION, \
    name_parts, compare_names
import codecs
import nltk
import re

                    if NLTK_VERSION >= 3:
                        c.set_label('LOCATION')
                    else:
                        c.node = 'LOCATION'
            yield c
        else:
            yield chunk


def _entity_types_and_locations(sentences):
    entities_by_type = defaultdict(list)
    locations = {}
    for sentnum, sent in enumerate(sentences, start=1):
        sent = pos_tag(tokenize(sent))
        chunks = chunk_named_entities(sent)
        chunks = [chunk for chunk in retag(chunks)]
        parser = nltk.RegexpParser(entity_chunk_grammar)
        parsed = parser.parse(chunks)
        for chunk in parsed:
            if not isinstance(chunk, nltk.tree.Tree):
                continue
            tokenized_name = tuple([c[0] for c in chunk.leaves()])
            e = KNOWN_ENTITIES.get(tokenized_name)
            if e:
                entity_key = EntityKey(tokenized_name=tokenized_name,
                    normalized_name=e['name'])
                entities_by_type[e['type']].append(entity_key)
            else:
                entity_key = EntityKey(tokenized_name=tokenized_name,
                    normalized_name=normalize_name(tokenized_name))

                if NLTK_VERSION >= 3: