<commit_before># -*- coding: utf-8 -*-
'''
 
                Copyright 2014 The MITRE Corporation.
 
  Licensed under the Apache License, Version 2.0 (the "License"); you may not
  use this file except in compliance with the License. You may obtain a copy of
  the License at
 
  http://www.apache.org/licenses/LICENSE-2.0
 
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  License for the specific language governing permissions and limitations under
  the License.
 
  =============================================================================

Created on Nov 19, 2014

@author: ubaldino

Demonstration of how to ingest a catalog of data to use with XTax.
Here JRC Names (published by JRC in the EU) is a collection of 
name/org entities that lists name variants in various languages.

REFERENCE: 
    https://ec.europa.eu/jrc/en/language-technologies/jrc-names
    Data is the resource file download, entities.gzip

USAGE:
    PYTHONPATH=<...include libs according to README>
    python jrcnames.py  /my/data/JRCNames/entities.txt   http://localhost:7000/solr/taxcat

    # python jrcnames.py  /patht/to/entities.txt   <solr url>

'entities.txt' file is a simple, TAB delimited file containing a
universal ID, entity type, language and name variant.

    1000595 P       u       Dani+Ábalo
    
The TaxCat / XTax strategy here is to use the following mapping:

    catalog  = JRC
    taxnode =  name type + primaryname
    phrase = name variant
    { 
      catalog = JRC,
      taxnode = 'Person.Dani Abalo',
      phrase =  '<NAME>'
      tags  = [ 'jrc=1000595', 'lang=u' ]
      
    }
      
Apparently lang = 'u' implies the name is universal in all languages, as spelled.
Where possible the first universal name variant found will act as the taxon primary name, 
creating the taxnode value.

'''

from opensextant.TaxCat import Taxon, TaxCatalogBuilder
from opensextant.CommonsUtils import is_ascii

# distinct primary names act as a JRC root entry; all others would be variants
#    That is just the convention here for demo sake.
nameset = {}

# ID map -- distinct JRC IDs must be reproducible and unique for each variant.
idset = {}

# Ambiguous phrases or noisy ones I would prefer not to tag, so
# they are marked valid=false, but retained in the names taxonomy.
# These are all valid JRC entities, however they do not seem to add value due to their
# popular use or ambiguity.  On very rare occasions there are instances of person names
# That coincide with popular place names.  The person name is marked is not valid for tagging.
NOISE = set(
        [
         'times',
         'the sun',
         'the times',
         'news agency',
         'daily news',
         'press agency',
         'military intelligence',
         'international studies',
         'international trade',
         'will meet',
         'the nation',
         'facebook',
         'google',
         'internet explorer',
         'yahoo',
         'twitter',
         'youtube',
         'they are',
         'our own',
         'reach',
         'armed forces',
         'canal',
         'the age',
         'read more',
         'presedential office',
         'set fire',
         'emergency',
         'nature',
         'status quo',
         'the independent',
         'gross domestic product',
         'privacy policy',
         'adobe reader',
         'guiding principles',
         'lessons learned',
         'better life',
         u'san diegó',  # not San Diego
         'san diego',  # not San Diego
         u'san franciscó',
         'san francisco',
         'corpus christi',
         u'nuevo león',
         u'nuevo léon',
         'nuevo leon',
         'san pedro',
         'umm qasr'
        ])

# Fixes are any entries that need to be remapped to entity type, p, o, etc. 
#
FIXES = {
         # Somehow this entry was marked as a "p" (Person)
         # So it is remapped to "-" (Thing, default)
         'Improvised Explosive Device' : '-'
         }

apply_default_fixes = True


def check_validity(e):
    '''
    @param e: a JRCEntity 
    '''
    phr = e.phrase.lower().replace(',', ' ')
    if phr in NOISE:
        e.is_valid = False
    return


PLACE_ENDING_FIXES = set(['province', 'island', 'islands', 'district', 'peninsula','valley',
                          'territory', 'county', 'city', 'state', 'township', 'village',
                          'roads', 'avenue', 'avenida', 'prefecture', 'heights'])
PLACE_STARTING_FIXES = set(['city', 'spin', 'town' ])

# A large number of entities are marked as Person unnecessarily.
# Org fixes: find any of these terms in a phrase and declare the phrase an org, if not already.
ORG_FIXES = set(['tribunal', 'council', 'shura', 'group', 'committee', 'senate',  ' of state',
                 'departamento', 'department', 'transport', 'transit', 'musee', 'museum', 
                 'museums', 'organization', 'authority', 'enterprise', 'institute', 'inc', 'llc'])
    
class JRCEntity(Taxon):
    def __init__(self, eid, variant_id, etype, lang, primary_name, ename):
        Taxon.__init__(self)<commit_msg>allow phrases like xx springs, fort abc, etc. to pass as places if they are marked as orgs<commit_after># -*- coding: utf-8 -*-
'''
 
                Copyright 2014 The MITRE Corporation.
 
  Licensed under the Apache License, Version 2.0 (the "License"); you may not
  use this file except in compliance with the License. You may obtain a copy of
  the License at
 
  http://www.apache.org/licenses/LICENSE-2.0
 
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  License for the specific language governing permissions and limitations under
  the License.
 
  =============================================================================

Created on Nov 19, 2014

@author: ubaldino

Demonstration of how to ingest a catalog of data to use with XTax.
Here JRC Names (published by JRC in the EU) is a collection of 
name/org entities that lists name variants in various languages.

REFERENCE: 
    https://ec.europa.eu/jrc/en/language-technologies/jrc-names
    Data is the resource file download, entities.gzip

USAGE:
    PYTHONPATH=<...include libs according to README>
    python jrcnames.py  /my/data/JRCNames/entities.txt   http://localhost:7000/solr/taxcat

    # python jrcnames.py  /patht/to/entities.txt   <solr url>

'entities.txt' file is a simple, TAB delimited file containing a
universal ID, entity type, language and name variant.

    1000595 P       u       Dani+Ábalo
    
The TaxCat / XTax strategy here is to use the following mapping:

    catalog  = JRC
    taxnode =  name type + primaryname
    phrase = name variant
    { 
      catalog = JRC,
      taxnode = 'Person.<NAME>',
      phrase =  '<NAME>'
      tags  = [ 'jrc=1000595', 'lang=u' ]
      
    }
      
Apparently lang = 'u' implies the name is universal in all languages, as spelled.
Where possible the first universal name variant found will act as the taxon primary name, 
creating the taxnode value.

'''

from opensextant.TaxCat import Taxon, TaxCatalogBuilder
from opensextant.CommonsUtils import is_ascii

# distinct primary names act as a JRC root entry; all others would be variants
#    That is just the convention here for demo sake.
nameset = {}

# ID map -- distinct JRC IDs must be reproducible and unique for each variant.
idset = {}

# Ambiguous phrases or noisy ones I would prefer not to tag, so
# they are marked valid=false, but retained in the names taxonomy.
# These are all valid JRC entities, however they do not seem to add value due to their
# popular use or ambiguity.  On very rare occasions there are instances of person names
# That coincide with popular place names.  The person name is marked is not valid for tagging.
NOISE = set(
        [
         'times',
         'the sun',
         'the times',
         'news agency',
         'daily news',
         'press agency',
         'military intelligence',
         'international studies',
         'international trade',
         'will meet',
         'the nation',
         'facebook',
         'google',
         'internet explorer',
         'yahoo',
         'twitter',
         'youtube',
         'they are',
         'our own',
         'reach',
         'armed forces',
         'canal',
         'the age',
         'read more',
         'presedential office',
         'set fire',
         'emergency',
         'nature',
         'status quo',
         'the independent',
         'gross domestic product',
         'privacy policy',
         'adobe reader',
         'guiding principles',
         'lessons learned',
         'better life',
         u'san diegó',  # not San Diego
         'san diego',  # not San Diego
         u'san franciscó',
         'san francisco',
         'corpus christi',
         u'nuevo león',
         u'nuevo léon',
         'nuevo leon',
         'san pedro',
         'umm qasr'
        ])

# Fixes are any entries that need to be remapped to entity type, p, o, etc. 
#
FIXES = {
         # Somehow this entry was marked as a "p" (Person)
         # So it is remapped to "-" (Thing, default)
         'Improvised Explosive Device' : '-'
         }

apply_default_fixes = True


def check_validity(e):
    '''
    @param e: a JRCEntity 
    '''
    phr = e.phrase.lower().replace(',', ' ')
    if phr in NOISE:
        e.is_valid = False
    return


PLACE_ENDING_FIXES = set(['province', 'island', 'islands', 'district', 'peninsula','valley',
                          'territory', 'county', 'city', 'state', 'township', 'village',
                          'roads', 'avenue', 'avenida', 'prefecture', 'heights', 'springs', 'spring',
                          'airport', 'aeropuerto', 'aeroporto', 'station' ])

PLACE_STARTING_FIXES = set(['city', 'spin', 'town', 'fort'])

# A large number of entities are marked as Person unnecessarily.
# Org fixes: find any of these terms in a phrase and declare the phrase an org, if not already.
ORG_FIXES = set(['tribunal', 'council', 'shura', 'group', 'committee', 'senate',  ' of state',
                 'departamento', 'department', 'transport', 'transit', 'musee', 'museum', 
                 'museums', 'organization', 'authority', 'enterprise', 'institute', 'inc', 'llc'])
    
class JRCEntity(Taxon):
    def __init__(self, eid, variant_id, etype, lang, primary_name, ename):
        Taxon.__init__(self)
{'fraction_non_alphanumeric': 0.11395479170558566, 'fraction_numerical': 0.00616476742013824, 'mean_word_length': 2.4921722113502933, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1063685', 'n_tokens_mistral': 3216, 'n_tokens_neox': 3058, 'n_words': 1178}