<commit_before># Natural Language Toolkit: Relation Extraction
#
# Copyright (C) 2001-2009 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://www.nltk.org/>
# For license information, see LICENSE.TXT

"""
Code for extracting relational triples from the ieer and conll2002 corpora.

Relations are stored internally as dictionaries ('reldicts'). 

The two serialization outputs are I{rtuple} and I{clause}. 
   - An I{rtuple} is a tuple of the form C{(subj, filler, obj)}, 
     where C{subj} and C{obj} are pairs of Named Entity mentions, and C{filler} is the string of words   
     occurring between C{sub} and C{obj} (with no intervening NEs). Strings are printed via C{repr()} to
     circumvent locale variations in rendering utf-8 encoded strings.
   - A I{clause} is an atom of the form C{relsym(subjsym, objsym)}, 
     where the relation, subject and object have been canonicalized to single strings.

"""

# todo: get a more general solution to canonicalized symbols for clauses -- maybe use xmlcharrefs?

from nltk import defaultdict

from string import join
import re
import htmlentitydefs
from itertools import ifilter

# Dictionary that associates corpora with NE classes
NE_CLASSES = {
    'ieer': ['LOCATION', 'ORGANIZATION', 'PERSON', 'DURATION', 
            'DATE', 'CARDINAL', 'PERCENT', 'MONEY', 'MEASURE'],
    'conll2002': ['LOC', 'PER', 'ORG']<commit_msg>fixed import bug (google code issue 29)<commit_after># Natural Language Toolkit: Relation Extraction
#
# Copyright (C) 2001-2009 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://www.nltk.org/>
# For license information, see LICENSE.TXT

"""
Code for extracting relational triples from the ieer and conll2002 corpora.

Relations are stored internally as dictionaries ('reldicts'). 

The two serialization outputs are I{rtuple} and I{clause}. 
   - An I{rtuple} is a tuple of the form C{(subj, filler, obj)}, 
     where C{subj} and C{obj} are pairs of Named Entity mentions, and C{filler} is the string of words   
     occurring between C{sub} and C{obj} (with no intervening NEs). Strings are printed via C{repr()} to
     circumvent locale variations in rendering utf-8 encoded strings.
   - A I{clause} is an atom of the form C{relsym(subjsym, objsym)}, 
     where the relation, subject and object have been canonicalized to single strings.

"""

# todo: get a more general solution to canonicalized symbols for clauses -- maybe use xmlcharrefs?

from nltk.compat import defaultdict

from string import join
import re
import htmlentitydefs
from itertools import ifilter

# Dictionary that associates corpora with NE classes
NE_CLASSES = {
    'ieer': ['LOCATION', 'ORGANIZATION', 'PERSON', 'DURATION', 
            'DATE', 'CARDINAL', 'PERCENT', 'MONEY', 'MEASURE'],
    'conll2002': ['LOC', 'PER', 'ORG']
{'fraction_non_alphanumeric': 0.1026099392205935, 'fraction_numerical': 0.012870933142652842, 'mean_word_length': 4.200743494423792, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '199974', 'n_tokens_mistral': 909, 'n_tokens_neox': 847, 'n_words': 349}