<jupyter_start><jupyter_text>Named Entity Recognition based on Language Models, PoS tagging and Subwords<jupyter_code>%%capture
!pip install --upgrade pip
!pip install flair
# PoS tagger

from flair.data import Sentence
from flair.models import SequenceTagger
tagger = SequenceTagger.load("flair/pos-english")

# NER model

from transformers import AutoTokenizer, AutoModelForTokenClassification
from transformers import pipeline

tokenizer = AutoTokenizer.from_pretrained("dslim/bert-base-NER-uncased")
model = AutoModelForTokenClassification.from_pretrained("dslim/bert-base-NER-uncased")

nlp = pipeline("ner", model=model, tokenizer=tokenizer)
def get_pos_entities(text,category):
    # make example sentence
    sentence = Sentence(text)

    # predict NER tags
    tagger.predict(sentence)

    # print sentence
    #print(sentence)
    # iterate over entities and print
    entities = []
    current_entity = ""
    for t in sentence.tokens:
        for label in t.annotation_layers.keys():
            text = t.text
            label = t.get_labels(label)[0].value   
            if (label == category):
                if (current_entity == ""):
                    current_entity += text
                else:
                    current_entity += " " + text
            elif len(current_entity) > 0:
                entities.append(current_entity)
                current_entity = ""
          
    if (len(current_entity)>0):
        entities.append(current_entity)
    return entities

r = get_pos_entities("who's a kung fu star from hong kong","NN")
print(r)
def get_entities(text):
    entities = []
    entity = ""
    index = -1
    offset = -1
    for token in nlp(text):
        if (index == -1):
            index = token['index']
            offset = token['start']
        word = token['word']
        if (word[0] == '#'):
            word = token['word'].replace("#","")
            
        if (token['start']== offset):
            entity += word
        elif (token['index']-index < 2):
            entity += " " + word
        else:
            entities.append(entity)
            entity = word
        index = token['index']
        offset = token['end']
        
    if (len(entity) > 0):    
        entities.append(entity)
    if (len(entities) == 0):
        cardinal_entities =  get_pos_entities(text,"CD")
        if (len(cardinal_entities)>0):
            return cardinal_entities
        noun_entities =  get_pos_entities(text,"NN")
        if (len(noun_entities)>0):
            return noun_entities
        
    return entities

r = get_entities("which city did carl-alfred schumacher die")
print(r)<jupyter_output>['<NAME>']<jupyter_text>Wikidata Entity Linking based on MediaWiki APIThe MediaWiki Action API is a web service that allows access to some wiki-features like authentication, page operations, and search. It can provide meta information about the wiki and the logged-in user.action=wbsearchentitiesSearches for entities using labels and aliases.Returns a label and description for the entity in the user language if possible. Returns details of the matched term. The matched term text is also present in the aliases key if different from the display label.<jupyter_code>import requests
def get_wikidata_candidates(label):
    candidates = []
    if (label==""):
        return candidates
    # type: One of the following values: form, form, item, lexeme, property, sense, sense
    query_path = "https://www.wikidata.org/w/api.php?action=wbsearchentities&search=QUERY_TEXT&language=en&limit=10&type=item&format=json"
    r = requests.get(query_path.replace("QUERY_TEXT",label))
    
    for answer in r.json()['search']:
        candidate = {
            'label': answer['display']['label']['value'],
            'id':answer['id']
#            'description' : answer['display']['description']
        }
        candidates.append(candidate)
    return candidates

r = get_wikidata_candidates("Berlín")
print(r)<jupyter_output>[{'label': 'Berlin', 'id': 'Q64'}, {'label': 'Berlin', 'id': 'Q821244'}, {'label': 'Berlin', 'id': 'Q614184'}, {'label': 'Berlín', 'id': 'Q582242'}, {'label': '1936 Summer Olympics', 'id': 'Q8150'}, {'label': 'Berlín', 'id': 'Q821971'}, {'label': 'Berlín', 'id': 'Q62270519'}, {'label': 'Technical University of Berlin', 'id': 'Q51985'}, {'label': 'Mitte', 'id': 'Q163966'}, {'label': 'Berlín', 'id': 'Q20274855'}]<jupyter_text>DBpedia Entity Linking based on DBpedia Lookup serviceThe DBpedia Lookup is an entity retrieval service for Linked Data. It provides a straightforward solution for the frequent use case of resolving keywords and natural language to related resource identifiers in the DBpedia knowledge graph. Related means that either the label or abstract of a resource matches, or an anchor text that was frequently used in Wikipedia to refer to a specific resource matches (e.g. the resource http://dbpedia.org/resource/United_States can be looked up by the string “USA”). So whether you need an auto-complete service for your RDF application, Linked Data enhancements for your CSV tables or simply a way to retrieve specific DBpedia identifiers – the DBpedia Lookup is for you!As a part of the DBpedia Technology Stack the DBpedia Lookup can be deployed conveniently via Docker and works well with DBpedia Databus Collections. The DBpedia Lookup uses an Apache Lucene Index for resource indexing and retrieval and provides a web interface for querying.<jupyter_code>import requests
def get_dbpedia_candidates(label):
    candidates = []
    if (label==""):
        return candidates
    # type: One of the following values: form, form, item, lexeme, property, sense, sense
    query_path = "http://lookup.dbpedia.org/api/search?query=QUERY_TEXT&maxResults=10&type=class&format=json"
    r = requests.get(query_path.replace("QUERY_TEXT",label))
    for answer in r.json()['docs']:
      if ('label' in answer) and ('resource' in answer):
        candidate = {
            'label': answer['label'][0].replace("<B>","").replace("</B>",""),
            'id':answer['resource'][0].split("http://dbpedia.org/resource/")[1]
#            'description' : answer['display']['description']
        }
        candidates.append(candidate)
    return candidates

r = get_dbpedia_candidates("Berlín")
print(r)<jupyter_output>[{'label': 'Berlin', 'id': 'Berlin'}, {'label': 'Bergen', 'id': 'Bergen'}, {'label': 'Tennis Borussia Berlin', 'id': 'Tennis_Borussia_Berlin'}, {'label': '1. FC Union Berlin', 'id': '1._FC_Union_Berlin'}, {'label': 'West Berlin', 'id': 'West_Berlin'}, {'label': 'East Berlin', 'id': 'East_Berlin'}, {'label': 'Berlín, Usulután', 'id': 'Berlín,_Usulután'}, {'label': 'Humboldt University of Berlin', 'id': 'Humboldt_University_of_Berlin'}, {'label': 'Hertha BSC', 'id': 'Hertha_BSC'}, {'label': '<NAME>', 'id': 'Carlos_Berlín_Montero'}]<jupyter_text>Wikidata Simple-Question with AnswersWe only consider questions with a single answer (i.e. predicate = 'P').<jupyter_code>import pandas as pd
df = pd.read_csv('wsq-labels.csv', index_col=0)
df.head()<jupyter_output><empty_output><jupyter_text>NER Task<jupyter_code># initialize entities column
entities = []
index = 0
rows = df.to_numpy().tolist()
for row in rows[index:]:
    question = row[3] #question
    print(index,":",question)
    question_entities = get_entities(question)
    entities.append(question_entities)
    index += 1
print(len(entities),"questions analyzed!")
df['entity']=entities
df.to_csv('wsq-entities.csv')
df.head()<jupyter_output><empty_output><jupyter_text>Entity Linking Task  Wikidata Entity Linking<jupyter_code># initialize wikidata entities column
wikidata_items = []
index = 0
rows = df.to_numpy().tolist()
for row in rows[index:]:
    items = []
    q_entities = row[7] #entity
    print(index,":",q_entities)
    for entity in q_entities:
      for item in get_wikidata_candidates(entity):
          items.append(item['id'])
    wikidata_items.append(items)    
    index += 1
print(len(wikidata_items),"questions analyzed!")
df['wiki_items']=wikidata_items
df.to_csv('wsq-wikidata.csv')
df.head()<jupyter_output><empty_output><jupyter_text>DBpedia Entity Linking<jupyter_code># initialize dbpedia entities column
dbpedia_items = []
index = 0
for row in rows[index:]:
    items = []
    q_entities = row[7] #entity
    print(index,":",q_entities)
    for entity in q_entities:
      for item in get_dbpedia_candidates(entity):
          items.append(item['id'])    
    dbpedia_items.append(items)
    index += 1
print(len(dbpedia_items),"questions analyzed!")
df['dbpedia_items']=dbpedia_items
df.to_csv('wsq-dbpedia.csv')
df.head()<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>%%capture
!pip install unidecode
def evaluate_labels(true_list,pred_list):
  index = 0
  tp, tn, fp, fn = 0, 0, 0, 0
  empty_values = 0
  for l1 in true_list:
    predicted_values = pred_list[index]
    if (len(predicted_values) == 0):
      empty_values += 1
    elif (l1[0] not in predicted_values):
      fn += 1
    else:  
      for l2 in predicted_values:
        if (l2 in l1):
          tp += 1
        else:
          fp += 1
    index += 1
    if (index % 1000 == 0):
      print(index)
  print("Total:",index)
  print("Empty Values:",empty_values)
  print("TruePositives:",tp,"TrueNegatives:",tn,"FalsePositives:",fp,"FalseNegatives:",fn)
  precision = tp / (fp + tp)
  print("Precission:",precision)
  recall = tp / (fn + tp)
  print("Recall:",recall)
  f1 = 2 * ((precision*recall)/(precision+recall))
  print("F1:",f1)
my_df = df[df.predicate.str.contains('P',case=False)]
my_df.head()
# NER Performance
import unidecode
y_true = [[unidecode.unidecode(i).lower().replace(" ","_")] for i in my_df['subject_label']]
y_pred = []
for entities in my_df['entity']:
  norm_entities = []
  for entity in entities:
    norm_entities.append(unidecode.unidecode(entity).lower().replace(" ","_"))
  y_pred.append(norm_entities)
evaluate_labels(y_true,y_pred)
# Wikidata Entity Linking Performance
y_true = [[i] for i in my_df['subject']]
y_pred = my_df['wiki_items'].tolist()
evaluate_labels(y_true,y_pred)<jupyter_output>1000
2000
3000
4000
Total: 4296
Empty Values: 110
TruePositives: 3382 TrueNegatives: 0 FalsePositives: 12089 FalseNegatives: 815
Precission: 0.21860254670027793
Recall: 0.8058136764355492
F1: 0.3439088875330486
{'fraction_non_alphanumeric': 0.13312663501598682, 'fraction_numerical': 0.02015308594128476, 'mean_word_length': 3.9434865900383143, 'pattern_counts': {'":': 3, '<': 34, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '477290', 'license': 'Apache-2.0', 'n_tokens_mistral': 3441, 'n_tokens_neox': 3289, 'n_words': 865, 'path': 'notebooks/backup/wikidata_simple_questions_NER.ipynb', 'repo_name': 'librairy/extractiveQA-over-KG', 'size': '10352'}