<jupyter_start><jupyter_text>[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/enterprise/healthcare/Disambiguation.ipynb)<jupyter_code>import json

with open('keys.json') as f:
    license_keys = json.load(f)

license_keys.keys()

import os

# Install java
! apt-get update -qq
! apt-get install -y openjdk-8-jdk-headless -qq > /dev/null

secret = license_keys.get("secret",license_keys.get('SPARK_NLP_SECRET', ""))
spark_version = os.environ.get("SPARK_VERSION", license_keys.get("SPARK_VERSION","2.4"))
version = license_keys.get("version",license_keys.get('SPARK_NLP_PUBLIC_VERSION', ""))
jsl_version = license_keys.get("jsl_version",license_keys.get('SPARK_NLP_VERSION', ""))

os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["PATH"] = os.environ["JAVA_HOME"] + "/bin:" + os.environ["PATH"]
! java -version

os.environ['SPARK_NLP_LICENSE'] = license_keys['SPARK_NLP_LICENSE']
os.environ['JSL_OCR_LICENSE'] = license_keys['JSL_OCR_LICENSE']
os.environ['AWS_ACCESS_KEY_ID']= license_keys['AWS_ACCESS_KEY_ID']
os.environ['AWS_SECRET_ACCESS_KEY'] = license_keys['AWS_SECRET_ACCESS_KEY']

print(spark_version, version, jsl_version)

! python -m pip install "pyspark==$spark_version".*
! python -m pip install --upgrade spark-nlp-jsl==$jsl_version  --extra-index-url https://pypi.johnsnowlabs.com/$secret

import sparknlp
import sparknlp_jsl
from sparknlp.base import *
from sparknlp.annotator import *
from sparknlp_jsl.annotator import *
from pyspark.ml import Pipeline
from pyspark.sql import SparkSession

print (sparknlp.version())
print (sparknlp_jsl.version())

spark = sparknlp_jsl.start(secret, gpu=False, spark23=(spark_version[:3]=="2.3"))
# Sample data
text = "The show also had a contestant named <NAME> " \
               + "who later defeated <NAME> on the way to become Female Vocalist Champion in the 1989 edition of Star Search in the United States. "
data = spark.createDataFrame([
            [text]]) \
            .toDF("text").cache()

# Preprocessing pipeline
da = DocumentAssembler().setInputCol("text").setOutputCol("document")
sd = SentenceDetector().setInputCols("document").setOutputCol("sentence")
tk = Tokenizer().setInputCols("sentence").setOutputCol("token")
emb = WordEmbeddingsModel.pretrained().setOutputCol("embs")
semb = SentenceEmbeddings().setInputCols("sentence","embs").setOutputCol("sentence_embeddings")
ner = NerDLModel.pretrained().setInputCols("sentence","token","embs").setOutputCol("ner")
nc = NerConverter().setInputCols("sentence","token","ner").setOutputCol("ner_chunk").setWhiteList(["PER"])
disambiguator = NerDisambiguator() \
    .setS3KnowledgeBaseName("i-per") \
    .setInputCols("ner_chunk", "sentence_embeddings") \
    .setOutputCol("disambiguation") \
    .setNumFirstChars(5)
pl = Pipeline().setStages([da,sd,tk,emb,semb,ner,nc,disambiguator])
data = pl.fit(data).transform(data)
data.selectExpr("explode(disambiguation)").show(10, False)<jupyter_output>glove_100d download started this may take some time.
Approximate size to download 145.3 MB
[OK!]
ner_dl download started this may take some time.
Approximate size to download 13.6 MB
[OK!]
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|col                                                                                                                                                                                                   [...]
{'fraction_non_alphanumeric': 0.2734874569601574, 'fraction_numerical': 0.007132316773241515, 'mean_word_length': 2.248402555910543, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '62656', 'license': 'Apache-2.0', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1135, 'n_words': 191, 'path': 'jupyter/enterprise/healthcare/Disambiguation.ipynb', 'repo_name': 'kaniska/spark-nlp-workshop', 'size': '4084'}