<jupyter_start><jupyter_text>Example of usage ImageAdaptiveThresholding and  ImageAdaptiveBinarizer transformers  Install spark-ocr python packgeNeed specify path to `spark-ocr-assembly-[version].jar` or `secret`<jupyter_code>secret = ""
license = ""
version = secret.split("-")[0]
spark_ocr_jar_path = "../../target/scala-2.11"
# install from PYPI using secret
#%pip install spark-ocr==$version+spark30 --extra-index-url=https://pypi.johnsnowlabs.com/$secret --upgrade
#or install from local path
#%pip install ../../python/dist/spark-ocr-3.9.0+spark30.tar.gz<jupyter_output>Processing /Users/nmelnik/IdeaProjects/spark-ocr/python/dist/spark-ocr-3.0.0.spark30.tar.gz
Requirement already satisfied: numpy==1.17.4 in /Users/nmelnik/.local/share/virtualenvs/OcrMissedPages-W3gpzUuJ/lib/python3.7/site-packages (from spark-ocr===3.0.0.spark30) (1.17.4)
Requirement already satisfied: pillow==6.2.1 in /Users/nmelnik/.local/share/virtualenvs/OcrMissedPages-W3gpzUuJ/lib/python3.7/site-packages (from spark-ocr===3.0.0.spark30) (6.2.1)
Requirement already satisfied: py4j==0.10.9 in /Users/nmelnik/.local/share/virtualenvs/OcrMissedPages-W3gpzUuJ/lib/python3.7/site-packages (from spark-ocr===3.0.0.spark30) (0.10.9)
Requirement already satisfied: pyspark==3.0.1 in /Users/nmelnik/.local/share/virtualenvs/OcrMissedPages-W3gpzUuJ/lib/python3.7/site-packages (from spark-ocr===3.0.0.spark30) (3.0.1)
Requirement already satisfied: python-levenshtein==0.12.0 in /Users/nmelnik/.local/share/virtualenvs/OcrMissedPages-W3gpzUuJ/lib/python3.7/site-packages (from spark-ocr===3.0.0.spark[...]<jupyter_text>Initialization of spark session<jupyter_code>from sparkocr import info
info()
from pyspark.sql import SparkSession
from sparkocr import start
import sys
import os

if license:
    os.environ['JSL_OCR_LICENSE'] = license

spark = start(secret=secret, jar_path=spark_ocr_jar_path)
spark<jupyter_output><empty_output><jupyter_text>Import OCR transformers<jupyter_code>from pyspark.ml import PipelineModel

from sparkocr.transformers import *
from sparkocr.enums import *
from sparkocr.utils import display_image<jupyter_output><empty_output><jupyter_text>Define OCR transformers and pipeline<jupyter_code>binary_to_image = BinaryToImage() \
    .setInputCol("content") \
    .setOutputCol("image")

ocr = ImageToText() \
    .setInputCol("corrected_image") \
    .setOutputCol("text") \
    .setConfidenceThreshold(70) \
    .setIgnoreResolution(False) \
    .setPageIteratorLevel(PageIteratorLevel.SYMBOL)<jupyter_output><empty_output><jupyter_text>Read PDF document as binary file<jupyter_code>import pkg_resources
image_example = "data/images/text_with_noise.png"
image_example_df = spark.read.format("binaryFile").load(image_example).cache()<jupyter_output><empty_output><jupyter_text>Gaussian local tresholding<jupyter_code>adaptive_thresholding = ImageAdaptiveThresholding() \
    .setInputCol("image") \
    .setOutputCol("corrected_image") \
    .setBlockSize(35) \
    .setOffset(90) \
    .setKeepInput(True)

pipeline = PipelineModel(stages=[
    binary_to_image,
    adaptive_thresholding,
    ocr
])

for r in pipeline.transform(image_example_df).select("image", "corrected_image", "text").collect():
    display_image(r.image)
    display_image(r.corrected_image)
    print(f"Detected text:\n{r.text}")<jupyter_output>Image:
        origin: file:/Users/nmelnik/IdeaProjects/spark-ocr/workshop/jupyter/data/images/text_with_noise.png 
        width: 1095
        height: 134
        mode: 10<jupyter_text>Sauvola tresholding<jupyter_code>adaptive_binarization = ImageAdaptiveBinarizer()
adaptive_binarization.setInputCol("image")
adaptive_binarization.setOutputCol("corrected_image")
adaptive_binarization.setWidth(20)
adaptive_binarization.setScale(3.5)
adaptive_binarization.setMethod(TresholdingMethod.SAUVOLA)

pipeline = PipelineModel(stages=[
    binary_to_image,
    adaptive_binarization,
    ocr
])

for r in pipeline.transform(image_example_df).select("image", "corrected_image", "text").collect():
    display_image(r.image)
    display_image(r.corrected_image)
    print(f"Detected text:\n{r.text}")<jupyter_output>Image:
        origin: file:/Users/nmelnik/IdeaProjects/spark-ocr/workshop/jupyter/data/images/text_with_noise.png 
        width: 1095
        height: 134
        mode: 10<jupyter_text>Gaussian local tresholding (Scala implementation)<jupyter_code>adaptive_binarization = ImageAdaptiveBinarizer()
adaptive_binarization.setInputCol("image")
adaptive_binarization.setOutputCol("corrected_image")
adaptive_binarization.setWidth(20)
adaptive_binarization.setScale(4)
adaptive_binarization.setMethod(TresholdingMethod.GAUSSIAN)

pipeline = PipelineModel(stages=[
    binary_to_image,
    adaptive_binarization,
    ocr
])

for r in pipeline.transform(image_example_df).select("image", "corrected_image", "text").collect():
    display_image(r.image)
    display_image(r.corrected_image)
    print(f"Detected text:\n{r.text}")<jupyter_output>Image:
        origin: file:/Users/nmelnik/IdeaProjects/spark-ocr/workshop/jupyter/data/images/text_with_noise.png 
        width: 1095
        height: 134
        mode: 10<jupyter_text>Otsu thresholding<jupyter_code>adaptive_binarization = ImageAdaptiveBinarizer()
adaptive_binarization.setInputCol("image")
adaptive_binarization.setOutputCol("corrected_image")
adaptive_binarization.setMethod(TresholdingMethod.OTSU)


pipeline = PipelineModel(stages=[
    binary_to_image,
    adaptive_binarization,
    ocr
])

for r in pipeline.transform(image_example_df).select("image", "corrected_image", "text").collect():
    display_image(r.image)
    display_image(r.corrected_image)
    print(f"Detected text:\n{r.text}")<jupyter_output>Image:
        origin: file:/Users/nmelnik/IdeaProjects/spark-ocr/workshop/jupyter/data/images/text_with_noise.png 
        width: 1095
        height: 134
        mode: 10
{'fraction_non_alphanumeric': 0.12291453864487573, 'fraction_numerical': 0.023152877085461356, 'mean_word_length': 5.542316258351893, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '143926', 'license': 'OML', 'n_tokens_mistral': 2220, 'n_tokens_neox': 2066, 'n_words': 273, 'path': 'jupyter/SparkOcrAdaptiveThresholding.ipynb', 'repo_name': 'JohnSnowLabs/spark-ocr-workshop', 'size': '5874'}