<jupyter_start><jupyter_text>![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)  12. Named Entity Disambiguation v3.0<jupyter_code>import os
import json
import string
import numpy as np
import pandas as pd


import sparknlp
import sparknlp_jsl
from sparknlp.base import *
from sparknlp.util import *
from sparknlp.annotator import *
from sparknlp_jsl.annotator import *
from sparknlp.pretrained import ResourceDownloader

from pyspark.sql import functions as F
from pyspark.ml import Pipeline, PipelineModel

pd.set_option('max_colwidth', 100)
pd.set_option('display.max_columns', 100)  
pd.set_option('display.expand_frame_repr', False)


print('sparknlp_jsl.version : ',sparknlp_jsl.version())

spark<jupyter_output><empty_output><jupyter_text>**Named Entity Disambiguation **is the task of mapping words of interest, such as names of persons, locations and companies, from an input text document to corresponding unique entities in a target Knowledge Base (KB). Words of interest are called Named Entities (NEs), mentions, or surface forms.<jupyter_code># Sample data
text = "The show also had a contestant named <NAME> " \
               + "who later defeated <NAME> on the way to become Female Vocalist Champion in the 1989 edition of Star Search in the United States. "
data = spark.createDataFrame([
            [text]]) \
            .toDF("text")

documentAssembler = DocumentAssembler()\
  .setInputCol("text")\
  .setOutputCol("document")

# Sentence Detector annotator, processes various sentences per line
sentenceDetector = SentenceDetector()\
  .setInputCols(["document"])\
  .setOutputCol("sentence")

# Tokenizer splits words in a relevant format for NLP
tokenizer = Tokenizer()\
  .setInputCols(["sentence"])\
  .setOutputCol("token")

word_embeddings = WordEmbeddingsModel.pretrained()\
  .setInputCols(["sentence", "token"])\
  .setOutputCol("embeddings")

sentence_embeddings = SentenceEmbeddings()\
  .setInputCols("sentence","embeddings")\
  .setOutputCol("sentence_embeddings")

ner_model = NerDLModel.pretrained() \
  .setInputCols(["sentence", "token", "embeddings"]) \
  .setOutputCol("ner")

ner_converter = NerConverter() \
  .setInputCols(["sentence", "token", "ner"]) \
  .setOutputCol("ner_chunk")\
  .setWhiteList(["PER"])


nlpPipeline = Pipeline(stages=[
    documentAssembler, 
    sentenceDetector,
    tokenizer,
    word_embeddings,
    sentence_embeddings,
    ner_model,
    ner_converter])


empty_data = spark.createDataFrame([[""]]).toDF("text")
model = nlpPipeline.fit(empty_data)

result = model.transform(data)
result.show()
disambiguator = NerDisambiguator() \
    .setS3KnowledgeBaseName("i-per") \
    .setInputCols("ner_chunk", "sentence_embeddings") \
    .setOutputCol("disambiguation") \
    .setNumFirstChars(5)
disambiguated = disambiguator.fit(result).transform(result)
disambiguated.selectExpr("explode(disambiguation)").show(10, False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11862678450033991, 'fraction_numerical': 0.006118286879673691, 'mean_word_length': 5.235169491525424, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '211464', 'license': 'Apache-2.0', 'n_tokens_mistral': 983, 'n_tokens_neox': 932, 'n_words': 220, 'path': 'tutorials/Certification_Trainings/Healthcare/databricks_notebooks/12. Named_Entity_Disambiguation_v3.0.ipynb', 'repo_name': 'gkovaig/spark-nlp-workshop', 'size': '2960'}