<jupyter_start><jupyter_text>![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/6.Clinical_Context_Spell_Checker.ipynb)  6. Context Spell Checker - Medical<jupyter_code>import json, os
from google.colab import files

if 'spark_jsl.json' not in os.listdir():
  license_keys = files.upload()
  os.rename(list(license_keys.keys())[0], 'spark_jsl.json')

with open('spark_jsl.json') as f:
    license_keys = json.load(f)

# Defining license key-value pairs as local variables
locals().update(license_keys)
os.environ.update(license_keys)
# Installing pyspark and spark-nlp
! pip install --upgrade -q pyspark==3.1.2 spark-nlp==$PUBLIC_VERSION

# Installing Spark NLP Healthcare
! pip install --upgrade -q spark-nlp-jsl==$JSL_VERSION  --extra-index-url https://pypi.johnsnowlabs.com/$SECRET
import json
import os

import sparknlp
import sparknlp_jsl

from sparknlp.base import *
from sparknlp.annotator import *
from sparknlp_jsl.annotator import *

from pyspark.ml import Pipeline
from pyspark.sql import SparkSession

params = {"spark.driver.memory":"16G",
          "spark.kryoserializer.buffer.max":"2000M",
          "spark.driver.maxResultSize":"2000M"}

spark = sparknlp_jsl.start(license_keys['SECRET'],params=params)

print("Spark NLP Version :", sparknlp.version())
print("Spark NLP_JSL Version :", sparknlp_jsl.version())

spark
documentAssembler = DocumentAssembler()\
    .setInputCol("text")\
    .setOutputCol("document")

tokenizer = RecursiveTokenizer()\
    .setInputCols(["document"])\
    .setOutputCol("token")\
    .setPrefixes(["\"", "(", "[", "\n"])\
    .setSuffixes([".", ",", "?", ")","!", "'s"])

spellModel = ContextSpellCheckerModel\
    .pretrained('spellcheck_clinical', 'en', 'clinical/models')\
    .setInputCols("token")\
    .setOutputCol("checked")
pipeline = Pipeline(
    stages = [
    documentAssembler,
    tokenizer,
    spellModel
    ])

empty_ds = spark.createDataFrame([[""]]).toDF("text")

lp = LightPipeline(pipeline.fit(empty_ds))<jupyter_output><empty_output><jupyter_text>Ok!, at this point we have our spell checking pipeline as expected. Let's see what we can do with it, see these errors,_She was **treathed** with a five day course of **amoxicilin** for a **resperatory** **truct** infection.__With pain well controlled on **orall** **meditation**, she was discharged to **reihabilitation** **facilitay**.__Her **adominal** examination is soft, nontender, and **nonintended**__The patient was seen by the **entocrinology** service and she was discharged on 40 units of **unsilin** glargine at night_      _No __cute__ distress_Check that some of the errors are valid English words, only by considering the context the right choice can be made.<jupyter_code>example = ["She was treathed with a five day course of amoxicilin for a resperatory truct infection . ",
           "With pain well controlled on orall meditation, she was discharged to reihabilitation facilitay.",
           "Her adominal examination is soft, nontender, and nonintended.",
           "The patient was seen by the entocrinology service and she was discharged on 40 units of unsilin glargine at night",
           "No cute distress",
          ]

for pairs in lp.annotate(example):
    print(list(zip(pairs['token'],pairs['checked'])))<jupyter_output>[('She', 'She'), ('was', 'was'), ('treathed', 'treated'), ('with', 'with'), ('a', 'a'), ('five', 'five'), ('day', 'day'), ('course', 'course'), ('of', 'of'), ('amoxicilin', 'amoxicillin'), ('for', 'for'), ('a', 'a'), ('resperatory', 'respiratory'), ('truct', 'tract'), ('infection', 'infection'), ('.', '.')]
[('With', 'With'), ('pain', 'pain'), ('well', 'well'), ('controlled', 'controlled'), ('on', 'on'), ('orall', 'oral'), ('meditation', 'medication'), (',', ','), ('she', 'she'), ('was', 'was'), ('discharged', 'discharged'), ('to', 'to'), ('reihabilitation', 'rehabilitation'), ('facilitay', 'facility'), ('.', '.')]
[('Her', 'Her'), ('adominal', 'abdominal'), ('examination', 'examination'), ('is', 'is'), ('soft', 'soft'), (',', ','), ('nontender', 'nontender'), (',', ','), ('and', 'and'), ('nonintended', 'nondistended'), ('.', '.')]
[('The', 'The'), ('patient', 'patient'), ('was', 'was'), ('seen', 'seen'), ('by', 'by'), ('the', 'the'), ('entocrinology', 'endocrinology'), ('service', 'se[...]
{'fraction_non_alphanumeric': 0.19449378330373002, 'fraction_numerical': 0.004440497335701598, 'mean_word_length': 5.300699300699301, 'pattern_counts': {'":': 3, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '127055', 'license': 'Apache-2.0', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1469, 'n_words': 377, 'path': 'tutorials/Certification_Trainings/Healthcare/6.Clinical_Context_Spell_Checker.ipynb', 'repo_name': 'iamvarol/spark-nlp-workshop', 'size': '4504'}